Hello all,
After optimizing rather large indexes on 10 shards (each index holds about 500,000 documents and is about 270-300 GB in size) we started getting intermittent TermInfosReader.get() ArrayIndexOutOfBounds exceptions. The exceptions sometimes seem to occur on all 10 shards at the same time and sometimes on one shard but not the others. We also sometimes get an "Internal Server Error" but that might be either a cause or an effect of the array index out of bounds. Here is the top part of the message:
java.lang.ArrayIndexOutOfBoundsException: -14127432
at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:246)
Any suggestions for troubleshooting would be appreciated.
Trace from tomcat logs appended below.
Tom Burton-West
-----------------------------------------------------------------------------------------------------------
Feb 5, 2010 8:09:02 AM org.apache.solr.common.SolrException log
SEVERE: java.lang.ArrayIndexOutOfBoundsException: -14127432
at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:246)
at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:218)
at org.apache.lucene.index.SegmentReader.docFreq(SegmentReader.java:943)
at org.apache.solr.search.SolrIndexReader.docFreq(SolrIndexReader.java:308)
at org.apache.lucene.search.IndexSearcher.docFreq(IndexSearcher.java:144)
at org.apache.lucene.search.Similarity.idf(Similarity.java:481)
at org.apache.lucene.search.TermQuery$TermWeight.<init>(TermQuery.java:44)
at org.apache.lucene.search.TermQuery.createWeight(TermQuery.java:146)
at org.apache.lucene.search.BooleanQuery$BooleanWeight.<init>(BooleanQuery.java:186)
at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:366)
at org.apache.lucene.search.Query.weight(Query.java:95)
at org.apache.lucene.search.Searcher.createWeight(Searcher.java:230)
at org.apache.lucene.search.Searcher.search(Searcher.java:171)
at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:651)
at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:545)
at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:581)
at org.apache.solr.search.SolrIndexSearcher.getDocListNC(SolrIndexSearcher.java:903)
at org.apache.solr.search.SolrIndexSearcher.getDocListC(SolrIndexSearcher.java:884)
at org.apache.solr.search.SolrIndexSearcher.search(SolrIndexSearcher.java:341)
at org.apache.solr.handler.component.QueryComponent.process(QueryComponent.java:176)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:195)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:131)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1299)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:338)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:241)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:172)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:548)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:108)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:174)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:875)
at org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:665)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:528)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:81)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:689)
at java.lang.Thread.run(Thread.java:619)
Feb 5, 2010 8:09:02 AM org.apache.solr.common.SolrException log
SEVERE: org.apache.solr.common.SolrException: Internal Server Error
Internal Server Error
request: http://solr-sdr-search-10:8081/serve-10/select
at org.apache.solr.client.solrj.impl.CommonsHttpSolrServer.request(CommonsHttpSolrServer.java:423)
at org.apache.solr.client.solrj.impl.CommonsHttpSolrServer.request(CommonsHttpSolrServer.java:242)
at org.apache.solr.handler.component.HttpCommComponent$1.call(SearchHandler.java:422)
at org.apache.solr.handler.component.HttpCommComponent$1.call(SearchHandler.java:394)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
