file-Indexing-Worker-thread.js

const urlParams=new URLSearchParams(location.search),debugMode="true"===urlParams.get("debug"),Phoenix={baseURL:"../"};importScripts("../phoenix/virtualfs.js"),importScripts("../utils/EventDispatcher.js"),importScripts("./WorkerComm.js"),importScripts("./language-service-worker-thread.js"),virtualfs.debugMode=debugMode,console.log("File indexing worker loaded in debug mode: ",debugMode),debugMode||(console.log=console.info=function(){});let projectCache=[],files,currentCrawlID=0,MAX_FILE_SIZE_TO_INDEX=16777216,currentCrawlIndex=0,crawlComplete=!1,crawlEventSent=!1,cacheStartTime=Date.now(),cacheSize=0,maxCacheSizeBytes=1073741824;function clearProjectCache(){projectCache=[]}function _statAsync(path){return new Promise((resolve,reject)=>{fs.stat(path,function(err,stats){err?reject(err):resolve(stats)})})}function _readFileAsync(path){return new Promise((resolve,reject)=>{fs.readFile(path,"utf8",function(err,data){err?reject(err):resolve(data)})})}async function getFilesizeInBytes(fileName){try{let stats;return(await _statAsync(fileName)).size||0}catch(ex){return console.log(ex),0}}function _isBinaryContent(content){if(!content)return!1;const nullIndex=content.indexOf("\0");return-1!==nullIndex&&nullIndex<8192}async function getFileContentsForFile(filePath){if(projectCache[filePath]||""===projectCache[filePath])return projectCache[filePath];try{let fileSize;if(await getFilesizeInBytes(filePath)<=MAX_FILE_SIZE_TO_INDEX){let contents=await _readFileAsync(filePath);_isBinaryContent(contents)?(console.log("file indexer: skipping binary file:",filePath),projectCache[filePath]=""):projectCache[filePath]=contents}else projectCache[filePath]=""}catch(ex){console.log(ex),projectCache[filePath]=""}return projectCache[filePath]}async function fileCrawler(crawlerID){if(!files||files&&0===files.length)return void setTimeout(()=>fileCrawler(crawlerID),1e3);const parallelRead=5;let readPromises=[],crawlingPaths=[];for(let i=0;i<5&&currentCrawlIndex<files.length;i++)crawlingPaths.push(files[currentCrawlIndex]),readPromises.push(getFileContentsForFile(files[currentCrawlIndex])),currentCrawlIndex++;try{let contents=await Promise.all(readPromises)||[];for(let content of contents)content&&content.length&&(cacheSize+=content.length)}catch(e){console.error("Something went wrong when indexing paths:",crawlingPaths,e)}if(currentCrawlID===crawlerID)if(maxCacheSizeBytes>0&&cacheSize>=maxCacheSizeBytes){if(crawlComplete=!0,!crawlEventSent){crawlEventSent=!0;let crawlTime=Date.now()-cacheStartTime;WorkerComm.triggerPeer("crawlComplete",{numFilesCached:currentCrawlIndex,cacheSizeBytes:cacheSize,crawlTimeMs:crawlTime,aborted:!0})}}else if(currentCrawlIndex<files.length)crawlComplete=!1,setTimeout(()=>fileCrawler(crawlerID));else{if(crawlComplete=!0,!crawlEventSent){crawlEventSent=!0;let crawlTime=Date.now()-cacheStartTime;WorkerComm.triggerPeer("crawlComplete",{numFilesCached:files.length,cacheSizeBytes:cacheSize,crawlTimeMs:crawlTime})}setTimeout(()=>fileCrawler(crawlerID),1e3)}}function _crawlProgressMessenger(){!crawlComplete&&files&&WorkerComm.triggerPeer("crawlProgress",{processed:currentCrawlIndex,total:files.length,cacheSizeBytes:cacheSize})}function initCache(fileList){console.log("file indexer: InitCache with num files: ",fileList.length),files=fileList,currentCrawlIndex=0,cacheSize=0,clearProjectCache(),crawlEventSent=!1,cacheStartTime=Date.now(),fileCrawler(currentCrawlID+=1),WorkerComm.triggerPeer("crawlStarted"),0===files.length&&WorkerComm.triggerPeer("crawlComplete",{numFilesCached:files.length,cacheSizeBytes:0,crawlTimeMs:0})}function removeFilesFromCache(updateObject){console.log("file Indexer Document removed",updateObject);let fileList=updateObject.fileList||[],filesInSearchScope=updateObject.filesInSearchScope||[],i=0;for(i=0;i<fileList.length;i++)delete projectCache[fileList[i]];function isNotInRemovedFilesList(path){return-1===filesInSearchScope.indexOf(path)}files=files?files.filter(isNotInRemovedFilesList):files}function addFilesToCache(updateObject){console.log("file Indexer Document add",updateObject);let fileList=updateObject.fileList||[],filesInSearchScope=updateObject.filesInSearchScope||[],i=0,changedFilesAlreadyInList=[],newFiles=[];for(i=0;i<fileList.length;i++)projectCache[fileList[i]]=null;function isInChangedFileList(path){return-1!==filesInSearchScope.indexOf(path)}function isNotAlreadyInList(path){return-1===changedFilesAlreadyInList.indexOf(path)}newFiles=(changedFilesAlreadyInList=files?files.filter(isInChangedFileList):[]).filter(isNotAlreadyInList),files||(console.error("file indexing worker: addFilesToCache called before initCache!!!"),files=[]),files.push(...newFiles)}function documentChanged(updateObject){console.log("documetn changed",updateObject),projectCache[updateObject.filePath]=updateObject.docContents}function setTauriWsFS(nodeWSURL){fs.setNodeWSEndpoint(nodeWSURL),fs.forceUseNodeWSEndpoint(!0)}function setCacheConfig(config){void 0!==config.maxCacheSizeBytes&&config.maxCacheSizeBytes>0&&(maxCacheSizeBytes=config.maxCacheSizeBytes)}setInterval(_crawlProgressMessenger,1e3),WorkerComm.setExecHandler("setCacheConfig",setCacheConfig),WorkerComm.setExecHandler("setTauriFSWS",setTauriWsFS),WorkerComm.setExecHandler("initCache",initCache),WorkerComm.setExecHandler("filesChanged",addFilesToCache),WorkerComm.setExecHandler("documentChanged",documentChanged),WorkerComm.setExecHandler("filesRemoved",removeFilesFromCache);