Method tokenizeInternal

src/google/adk/cli/browser/chunk-NALL4A3P.js:90–90

(r,n)

	 Or inspect the console log for details on how to resolve these issues.`)}),this.TRACE_INIT("clearRegExpParserCache",()=>{cm()}),this.TRACE_INIT("toFastProperties",()=>{ya(this)})})}tokenize(r,n=this.defaultMode){if(!Z(this.lexerDefinitionErrors)){let s=I(this.lexerDefinitionErrors,a=>a.message).join(`-----------------------
`);throw new Error(`Unable to Tokenize because Errors detected in definition of Lexer:
`+s)}return this.tokenizeInternal(r,n)}tokenizeInternal(r,n){let i,s,a,o,c,l,u,p,h,g,C,k,G,M,b,E=r,H=E.length,F=0,ye=0,dr=this.hasCustom?0:Math.floor(r.length/10),Je=new Array(dr),Qt=[],Kt=this.trackStartLines?1:void 0,$=this.trackStartLines?1:void 0,y=ym(this.emptyGroups),L=this.trackStartLines,O=this.config.lineTerminatorsPattern,T=0,x=[],A=[],_=[],U=[];Object.freeze(U);let N=!1,Y=ve=>{if(_.length===1&&ve.tokenType.PUSH_MODE===void 0){let We=this.config.errorMessageProvider.buildUnableToPopLexerModeMessage(ve);Qt.push({offset:ve.startOffset,line:ve.startLine,column:ve.startColumn,length:ve.image.length,message:We})}else{_.pop();let We=gr(_);x=this.patternIdxToConfig[We],A=this.charCodeToPatternIdxToConfig[We],T=x.length;let js=this.canModeBeOptimized[We]&&this.config.safeMode===!1;A&&js?N=!0:N=!1}};function Q(ve){_.push(ve),A=this.charCodeToPatternIdxToConfig[ve],x=this.patternIdxToConfig[ve],T=x.length,T=x.length;let We=this.canModeBeOptimized[ve]&&this.config.safeMode===!1;A&&We?N=!0:N=!1}Q.call(this,n);let de,ue=this.config.recoveryEnabled;for(;F<H;){l=null,h=-1;let ve=E.charCodeAt(F),We;if(N){let ot=vr(ve),_t=A[ot];We=_t!==void 0?_t:U}else We=x;let js=We.length;for(i=0;i<js;i++){de=We[i];let ot=de.pattern;u=null;let _t=de.short;if(_t!==!1?ve===_t&&(h=1,l=ot):de.isCustom===!0?(b=ot.exec(E,F,Je,y),b!==null?(l=b[0],h=l.length,b.payload!==void 0&&(u=b.payload)):l=null):(ot.lastIndex=F,h=this.matchLength(ot,r,F)),h!==-1){if(c=de.longerAlt,c!==void 0){l=r.substring(F,F+h);let pr=c.length;for(a=0;a<pr;a++){let Zt=x[c[a]],ft=Zt.pattern;if(p=null,Zt.isCustom===!0?(b=ft.exec(E,F,Je,y),b!==null?(o=b[0],b.payload!==void 0&&(p=b.payload)):o=null):(ft.lastIndex=F,o=this.match(ft,r,F)),o&&o.length>l.length){l=o,h=o.length,u=p,de=Zt;break}}}break}}if(h!==-1){if(g=de.group,g!==void 0&&(l=l!==null?l:r.substring(F,F+h),C=de.tokenTypeIdx,k=this.createTokenInstance(l,F,C,de.tokenType,Kt,$,h),this.handlePayload(k,u),g===!1?ye=this.addToken(Je,ye,k):y[g].push(k)),L===!0&&de.canLineTerminator===!0){let ot=0,_t,pr;O.lastIndex=0;do l=l!==null?l:r.substring(F,F+h),_t=O.test(l),_t===!0&&(pr=O.lastIndex-1,ot++);while(_t===!0);ot!==0?(Kt=Kt+ot,$=h-pr,this.updateTokenEndLineColumnLocation(k,g,pr,ot,Kt,$,h)):$=this.computeNewColumn($,h)}else $=this.computeNewColumn($,h);F=F+h,this.handleModes(de,Y,Q,k)}else{let ot=F,_t=Kt,pr=$,Zt=ue===!1;for(;Zt===!1&&F<H;)for(F++,s=0;s<T;s++){let ft=x[s],Bs=ft.pattern,Lo=ft.short;if(Lo!==!1?E.charCodeAt(F)===Lo&&(Zt=!0):ft.isCustom===!0?Zt=Bs.exec(E,F,Je,y)!==null:(Bs.lastIndex=F,Zt=Bs.exec(r)!==null),Zt===!0)break}if(G=F-ot,$=this.computeNewColumn($,G),M=this.config.errorMessageProvider.buildUnexpectedCharactersMessage(E,ot,G,_t,pr,gr(_)),Qt.push({offset:ot,line:_t,column:pr,length:G,message:M}),ue===!1)break}}return this.hasCustom||(Je.length=ye),{tokens:Je,groups:y,errors:Qt}}handleModes(r,n,i,s){if(r.pop===!0){let a=r.push;n(s),a!==void 0&&i.call(this,a)}else r.push!==void 0&&i.call(this,r.push)}updateTokenEndLineColumnLocation(r,n,i,s,a,o,c){let l,u;n!==void 0&&(l=i===c-1,u=l?-1:0,s===1&&l===!0||(r.endLine=a+u,r.endColumn=o-1+-u))}computeNewColumn(r,n){return r+n}createOffsetOnlyToken(r,n,i,s){return{image:r,startOffset:n,tokenTypeIdx:i,tokenType:s}}createStartOnlyToken(r,n,i,s,a,o){return{image:r,startOffset:n,startLine:a,startColumn:o,tokenTypeIdx:i,tokenType:s}}createFullToken(r,n,i,s,a,o,c){return{image:r,startOffset:n,endOffset:n+c-1,startLine:a,endLine:a,startColumn:o,endColumn:o+c-1,tokenTypeIdx:i,tokenType:s}}addTokenUsingPush(r,n,i){return r.push(i),n}addTokenUsingMemberAccess(r,n,i){return r[n]=i,n++,n}handlePayloadNoCustom(r,n){}handlePayloadWithCustom(r,n){n!==null&&(r.payload=n)}match(r,n,i){return r.test(n)===!0?n.substring(i,r.lastIndex):null}matchLength(r,n,i){return r.test(n)===!0?r.lastIndex-i:-1}}return t.SKIPPED="This marks a skipped Token pattern, this means each token identified by it will be consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace.",t.NA=/NOT_APPLICABLE/,t})();function Vr(t){return mf(t)?t.LABEL:t.name}function mf(t){return mt(t.LABEL)&&t.LABEL!==""}var SE="parent",Nm="categories",Cm="label",wm="group",Im="push_mode",_m="pop_mode",bm="longer_alt",Pm="line_breaks",Om="start_chars_hint";function mn(t){return kE(t)}function kE(t){let e=t.pattern,r={};if(r.name=t.name,$t(e)||(r.PATTERN=e),q(t,SE))throw`The parent property is no longer supported.
See: https://github.com/chevrotain/chevrotain/issues/564#issuecomment-349062346 for details.`;return q(t,Nm)&&(r.CATEGORIES=t[Nm]),Wr([r]),q(t,Cm)&&(r.LABEL=t[Cm]),q(t,wm)&&(r.GROUP=t[wm]),q(t,_m)&&(r.POP_MODE=t[_m]),q(t,Im)&&(r.PUSH_MODE=t[Im]),q(t,bm)&&(r.LONGER_ALT=t[bm]),q(t,Pm)&&(r.LINE_BREAKS=t[Pm]),q(t,Om)&&(r.START_CHARS_HINT=t[Om]),r}var Bt=mn({name:"EOF",pattern:He.NA});Wr([Bt]);function Kr(t,e,r,n,i,s,a,o){return{image:e,startOffset:r,endOffset:n,startLine:i,endLine:s,startColumn:a,endColumn:o,tokenTypeIdx:t.tokenTypeIdx,tokenType:t}}function xa(t,e){return Br(t,e)}var Hr={buildMismatchTokenMessage({expected:t,actual:e,previous:r,ruleName:n}){return`Expecting ${mf(t)?`--> ${Vr(t)} <--`:`token of type --> ${t.name} <--`} but found --> '${e.image}' <--`},buildNotAllInputParsedMessage({firstRedundant:t,ruleName:e}){return"Redundant input, expecting EOF but found: "+t.image},buildNoViableAltMessage({expectedPathsPerAlt:t,actual:e,previous:r,customUserDescription:n,ruleName:i}){let s="Expecting: ",o=`
but found: '`+pt(e).image+"'";if(n)return s+n+o;{let c=Ue(t,(h,g)=>h.concat(g),[]),l=I(c,h=>`[${I(h,g=>Vr(g)).join(", ")}]`),p=`one of these possible Token sequences:
${I(l,(h,g)=>`  ${g+1}. ${h}`).join(`