public class WsHashEntry { WsHashEntry *next; I1 *word; U4 type,use_cnt; I1 *source_link; DbgInfo *debug; I4 num; U4 hits; U2 last_vowel,last_consonant; I1 *glossary; }; class WsSubstruct { I4 hits, repeats; }; #define WS_SUBSTRUCTS_PER_STRUCT 8 class WsStruct { WsStruct *next,*last; I4 substructs, //-1 means WsMatrixSide entries; WsSubstruct subs[WS_SUBSTRUCTS_PER_STRUCT]; }; class WsMatrixSide { WsStruct *next,*last; I4 const_minus_one, //-1 used to header in queue row_hits,col_hits; }; #define WS_MAX_MATRIX_WORDS 0x8000 public class WsMatrix { I4 n; WsMatrixSide side[WS_MAX_MATRIX_WORDS]; }; public I4 ws_num_words,ws_num_matrix_words; SysHashTable *ws_hash_table=NewHashTable(4096); public I1 *ws_cur_word; public BoolU4 ws_disable=TRUE; #define NUM_WS_MATRICES 1 public WsMatrix ws_matrices[NUM_WS_MATRICES]; public WsHashEntry **ws_sorted_words; TssStruct *ws_task=NULL; class WsFilemaskEntry { WsFilemaskEntry *next; I1 *mask; LTDirEntry *flist; }; WsFilemaskEntry *ws_filemask_list=NULL, *ws_gloss_filemask_list=NULL; void WsPutMatrix(WsMatrix *m) { I8 i,j; WsMatrixSide *ms; WsStruct *tempw; coutln "n:",m->n; for (i=0;iside[i]; coutln "col:",i," rhits:",ms->row_hits," chits:",ms->col_hits; tempw=ms->next; while (tempw!=ms) { coutln " substructs:",tempw->substructs, " entries:",tempw->entries; for (j=0;jsubstructs;j++) { PrintF("(%04X %04X) ", tempw->subs[j].hits, tempw->subs[j].repeats); } CrLf; tempw=tempw->next; } } View; } WsHashEntry *WsAddHash(I1 *w) { WsHashEntry **tempw1,*tempw=FindHashEntry(w,ws_hash_table,HTT_WORD); if (tempw) { tempw->hits++; return tempw; } tempw1=FindHashBucket(w,ws_hash_table); tempw=AMAllocZ(sizeof(WsHashEntry)); tempw->word=ANewString(w); tempw->type=HTT_WORD; tempw->use_cnt=1; tempw->hits=1; tempw->num=-1; tempw->next=*tempw1; *tempw1=tempw; ws_num_words++; return tempw; } void WsInitSides(WsMatrix *m) { I8 i; WsStruct *tempa; m->n=0; for (i=0;iside[i].next=&m->side[i].next; m->side[i].last=&m->side[i].next; m->side[i].const_minus_one=-1; m->side[i].row_hits=0; m->side[i].col_hits=0; tempa=AMAlloc(sizeof(WsStruct)); tempa->subs[0].hits=0; tempa->subs[0].repeats=ws_num_matrix_words; tempa->entries=ws_num_matrix_words; tempa->substructs=1; InsQue(tempa,m->side[i].last); } } WsStruct *WsFindStruct(WsMatrix *m,I8 col,I8 row, I4 *row_base,I4 *substruct) { WsStruct *tempa; if (row>=ws_num_matrix_words || col>=ws_num_matrix_words) return NULL; if (row>1) { tempa=m->side[col].next; *row_base=0; while (row>=tempa->entries) { row-=tempa->entries; *row_base+=tempa->entries; tempa=tempa->next; } for (*substruct=0;row>=tempa->subs[*substruct].repeats;*substruct=*substruct+1) { row-=tempa->subs[*substruct].repeats; *row_base+=tempa->subs[*substruct].repeats; } } else { row=ws_num_matrix_words-row; tempa=m->side[col].last; *row_base=ws_num_matrix_words; while (row>tempa->entries) { row-=tempa->entries; *row_base-=tempa->entries; tempa=tempa->last; } for (*substruct=tempa->substructs-1;row>tempa->subs[*substruct].repeats; *substruct=*substruct-1) { row-=tempa->subs[*substruct].repeats; *row_base-=tempa->subs[*substruct].repeats; } *row_base-=tempa->subs[*substruct].repeats; } return tempa; } public I8 WsGetHits(WsMatrix *m,I8 col,I8 row) { I8 row_base,substruct; if (row>=ws_num_matrix_words || col>=ws_num_matrix_words) return 0; WsStruct *tempa=WsFindStruct(m,col,row, &row_base,&substruct); return tempa->subs[substruct].hits; } WsSubstruct *WsNext(WsSubstruct *s,WsStruct **parent) { I8 i=(s-&(*parent)->subs)/sizeof(WsSubstruct)+1; if (i>=(*parent)->substructs) { i=0; *parent=(*parent)->next; if ((*parent)->substructs==-1) return NULL; } return (*parent)->subs+i*sizeof(WsSubstruct); } WsSubstruct *WsLast(WsSubstruct *s,WsStruct **parent) { I8 i=(s-&(*parent)->subs)/sizeof(WsSubstruct)-1; if (i<0) { *parent=(*parent)->last; if ((*parent)->substructs==-1) return NULL; i=(*parent)->substructs-1; } return (*parent)->subs+i*sizeof(WsSubstruct); } void WsRemove(WsSubstruct *s,WsStruct *parent) { I8 i=(s-&parent->subs)/sizeof(WsSubstruct); if (i<0 || i>=WS_SUBSTRUCTS_PER_STRUCT || parent->substructs<1) return; if (parent->substructs==1) { RemQue(parent); AFree(parent); } else { parent->entries-=s->repeats; parent->substructs--; MemCpy(&parent->subs[i],&parent->subs[i+1], (parent->substructs-i)*sizeof(WsSubstruct)); } } WsSubstruct *WsInsert(WsSubstruct *s,WsStruct **parent,WsSubstruct *last) { WsStruct *tempa,*p=*parent; I8 j,k,i=(last-&p->subs)/sizeof(WsSubstruct)+1; if (i>=WS_SUBSTRUCTS_PER_STRUCT) { i=0; p=p->next; *parent=p; if (p->substructs==-1) { tempa=AMAlloc(sizeof(WsStruct)); InsQue(tempa,p->last); p=tempa; *parent=p; tempa->entries=0; tempa->substructs=0; } } if (p->substructs==WS_SUBSTRUCTS_PER_STRUCT) { tempa=p->next; if (tempa->substructs==-1 || tempa->substructs==WS_SUBSTRUCTS_PER_STRUCT) { tempa=AMAlloc(sizeof(WsStruct)); tempa->entries=0; tempa->substructs=0; InsQue(tempa,p); k=0; for (j=i;jsubstructs;j++) { MemCpy(&tempa->subs[k],&p->subs[j],sizeof(WsSubstruct)); tempa->entries+=tempa->subs[k++].repeats; tempa->substructs++; } p->entries-=tempa->entries; p->substructs-=tempa->substructs; } else { for (j=tempa->substructs-1;j>=0;j--) MemCpy(&tempa->subs[j+1],&tempa->subs[j],sizeof(WsSubstruct)); MemCpy(&tempa->subs[0], &p->subs[WS_SUBSTRUCTS_PER_STRUCT-1], sizeof(WsSubstruct)); p->entries-=tempa->subs[0].repeats; p->substructs--; tempa->entries+=tempa->subs[0].repeats; tempa->substructs++; for (j=WS_SUBSTRUCTS_PER_STRUCT-2;j>=i;j--) MemCpy(&p->subs[j+1], &p->subs[j],sizeof(WsSubstruct)); } } else { for (j=p->substructs-1;j>=i;j--) MemCpy(&p->subs[j+1], &p->subs[j],sizeof(WsSubstruct)); } p->entries+=s->repeats; p->substructs++; MemCpy(&p->subs[i],s,sizeof(WsSubstruct)); return &p->subs[i]; } WsSubstruct *WsCondense(WsSubstruct *s,WsStruct **parent) { WsSubstruct *temps,*temps_l,*temps_n; WsStruct *tempa_l,*tempa_n; if ((*parent)->substructs==-1) return NULL; temps=s; tempa_l=*parent; temps_l=WsLast(temps,&tempa_l); if (temps_l && temps_l->hits==temps->hits) { temps_l->repeats+=temps->repeats; tempa_l->entries+=temps->repeats; WsRemove(temps,*parent); *parent=tempa_l; temps=temps_l; } tempa_n=*parent; temps_n=WsNext(temps,&tempa_n); if (temps_n && temps_n->hits==temps->hits) { temps->repeats=temps->repeats+temps_n->repeats; (*parent)->entries=(*parent)->entries+temps_n->repeats; WsRemove(temps_n,tempa_n); } return temps; } I8 WsIncHits(WsMatrix *m,I8 col,I8 row) { I8 row_base,substruct,repeats; WsSubstruct sn,*temps,*temps1,*temps_l; WsStruct *tempa,*tempa1,*tempa_l; if (row>=ws_num_matrix_words || col>=ws_num_matrix_words) return 0; tempa=WsFindStruct(m,col,row,&row_base,&substruct); temps=&tempa->subs[substruct]; repeats=temps->repeats; if (repeats==1) { temps->hits++; temps=WsCondense(temps,&tempa); } else { if (row==row_base) { temps->repeats--; tempa->entries--; sn.hits=1; sn.repeats=1; tempa_l=tempa; if (!(temps_l=WsLast(&tempa->subs[substruct],&tempa_l))) { tempa_l=tempa->last; //header temps_l=&tempa_l->subs[WS_SUBSTRUCTS_PER_STRUCT-1]; } temps=WsInsert(&sn,&tempa_l,temps_l); temps=WsCondense(temps,&tempa_l); } else if (row==repeats-1+row_base) { temps->repeats--; tempa->entries--; sn.hits=1; sn.repeats=1; temps=WsInsert(&sn,&tempa,temps); temps=WsCondense(temps,&tempa); } else { tempa->entries-=repeats; sn.hits=temps->hits+1; sn.repeats=1; tempa1=tempa; temps1=temps; temps=WsInsert(&sn,&tempa1,temps1); temps1->repeats=row-row_base; tempa->entries+=temps1->repeats; sn.repeats=row_base+repeats-(row+1); sn.hits=temps1->hits; WsInsert(&sn,&tempa1,temps); } } m->n++; m->side[col].col_hits++; m->side[row].row_hits++; return temps->hits; } #define WS_FIFO_SIZE 2048 void WsAddSingleFile(LTDirEntry *tempd,BoolU4 compile) { I1 ch,ch1,*buf,*ptr,*ptr2; WsHashEntry **f_ptr,**f_ptr2, **fifo=MAllocZ(WS_FIFO_SIZE*sizeof(U4)); I8 i,j,fp=0,k=0; buf=ReadTextFile(tempd->full_name); ptr=buf; while (ch=*ptr++) { while (ch && !Bt(alpha_numeric_bitmap,ch)) ch=*ptr++; if (!ch) break; ptr--; ptr2=ptr; do ch=*ptr2++; while (Bt(alpha_numeric_bitmap,ch)); ptr2--; ch1=*ptr2; *ptr2=0; if (*ptr) { if (!compile) WsAddHash(ptr); else { f_ptr=fifo+fp<<2; *f_ptr=FindHashEntry(ptr,ws_hash_table,HTT_WORD); if ((*f_ptr)->num<0) *f_ptr=NULL; else { for (i=0;inum,(*f_ptr)->num); } } } fp=(fp+1)&(WS_FIFO_SIZE-1); } ptr=ptr2+1; } if (!ch1) break; *ptr2=ch1; } Free(fifo); Free(buf); } void WsAddGlossFile(LTDirEntry *tempd) { LtfEntry *ll; WsHashEntry *temph; Ltf *l=LtfRead(tempd->full_name); I1 *temp; ll=l->dummy.next; while (ll!=l) { if (ll->btype==LTFT_ANCHOR) { if (temph=FindHashEntry(ll->aux_string, ws_hash_table,HTT_WORD)) { temp=MSPrintF("FA:%s,%s", tempd->full_name,ll->aux_string); AFree(temph->glossary); temph->glossary=ANewString(temp); Free(temp); } } ll=ll->next; } LtfDel(l); } I8 WsSort(WsHashEntry **e1,WsHashEntry **e2) { I8 r; if (!(r=(*e2)->hits - (*e1)->hits)) r=StrCmp((*e1)->word,(*e2)->word); return r; } void WsAddFileList(LTDirEntry *tempm1,BoolU4 compile,BoolU4 gloss) { while (tempm1) { coutln tempm1->full_name; if (tempm1->attr & LT_ATTR_DIR) { if (tempm1->sub) WsAddFileList(tempm1->sub,compile,gloss); } else { if (gloss) WsAddGlossFile(tempm1); else WsAddSingleFile(tempm1,compile); } tempm1=tempm1->next; } } I8 WsSkipCrap(I1 *src,I8 len) { I8 j; j=len-1; while (j>=0) { if (Bt(alpha_numeric_bitmap,src[j])) break; else j--; } return j+1; } I8 WsPriorWordInString(I1 *src,I1 *dst,I8 len,I8 buf_size) { I8 i,j=0,k; BoolU4 cont=TRUE; i=len-1; while (i>=0 && cont) { if (!Bt(alpha_numeric_bitmap,src[i])) cont=FALSE; else i--; } if (i>=-1 && len>0) for (k=i+1;k=0 && nword; len=StrLen(s); if (len>ws_partial_len) Auto(s+ws_partial_len); } } void WsMan(I8 n,TssStruct *parent=NULL) { I1 *buf; WsHashEntry *tempw; SysHashEntry *temph; BoolU4 old_preempt=Preempt(OFF); n--; if (n>=0 && nglossary; if (temph=FindHashEntry(tempw->word,Fs->hash_table,HTT_ALL)) { if (temph->source_link) buf=temph->source_link; } if (buf) { buf=MSPrintF("Edit(\"%s\");",buf); Preempt(old_preempt); PopUp(buf,parent); Free(buf); } } } Preempt(old_preempt); } void WsAddFillin(I8 i,I8 j,I8 old_num_fillins) { BoolU4 cont; I8 k; if (ws_num_fillinsfillin_scores[ws_num_fillins-1] || i==fillin_scores[ws_num_fillins-1] && ws_sorted_words[j]->hits>fillin_hits[ws_num_fillins-1]) { cont=TRUE; for (k=ws_num_fillins-1;k>=old_num_fillins && cont;k--) if (i<=fillin_scores[k] || i==fillin_scores[k] && ws_sorted_words[j]->hits<=fillin_hits[k]) { cont=FALSE; k++; //offset k-- } else { fillin_scores[k+1]=fillin_scores[k]; fillin_matches[k+1]=fillin_matches[k]; fillin_hits[k+1]=fillin_hits[k]; } if (ws_num_fillinshits; fillin_matches[k+1]=j; } } void WsPutChoices(Ltf *l,Ltf *aux,Ltf *focus_l,LtfEntry *ll) { I8 i,j,k,data_col; I1 ch=0,*buf,*buf1,*src=NULL,*st; WsHashEntry *w2; U8 timeout_jiffies=jiffies+JIFFY_FREQ*20/1000; SysHashEntry *temph; TssStruct *tss; src=LtfGetLine(focus_l,ll,&data_col); i=StrLen(src); buf=MAlloc(i+1); buf1=MAlloc(i+1); if (data_col==-1) data_col=StrLen(i); data_col=WsPriorWordInString(src,buf,data_col,256); ws_partial_len=StrLen(buf); data_col=WsSkipCrap(src,data_col); data_col=WsPriorWordInString(src,buf1,data_col,256); AFree(ws_cur_word); ws_cur_word=ANewString(buf); if (StrLen(buf1)) w2=FindHashEntry(buf1,ws_hash_table,HTT_WORD); else w2=NULL; Free(src); ws_num_fillins=0; if (w2 && w2->numword)) { i=WsGetHits(&ws_matrices[0],w2->num,ws_sorted_words[j]->num); WsAddFillin(i,j,0); } k=ws_num_fillins; for (j=0;jword)) WsAddFillin(0,j,k); if (ws_num_fillins) { LtfReset(l,TRUE); l->flags|=LTFF_MIN_SIZE; l->text_attribute=(LTBLUE<<4)+BLACK; aux->text_attribute=(LTBLUE<<4)+BLACK; Fs->text_attribute=l->text_attribute; Fs->border_attribute=l->text_attribute; LtfPrintF(l,"Word:%-20ts\r\n",ws_cur_word); for (i=0;iword; if (ws_sorted_words[fillin_matches[i]]->glossary) ch='#'; else ch=CH_SPACE; if (tss=sys_cur_focus_task) { temph=FindHashEntry(st,tss->hash_table,HTT_ALL); if (temph) { if (temph->source_link) ch='*'; } } LtfPrintF(l,"%cF%02d %-20ts\r\n",ch,i+1,st); } LtfRecalc(l); Bts(&Fs->crt_flags,CRTf_SHOW); WindowToTop(Fs); } Free(buf); Free(buf1); } void WsTsk() { TssStruct *focus_tss; U8 sc,ch; Ltf *l; LtfEntry *ll; Preempt(OFF); InitLocalTask; UseConsoleLtf(NULL); Fs->window_left=200; Fs->window_bottom=30; do { Sleep(500); if (!ws_disable) { focus_tss=sys_cur_focus_task; if (focus_tss) { l=focus_tss->cur_ltf; if (l && focus_tss!=Fs) { LtfRecalc(l); //TODO: is this needed? ll=l->cur_entry; if (ll) { if (ll==l) ll=ll->last; while (ll->last!=l && (ll->btype==LTFT_CR || ll->btype==LTFT_SOFT_CR)) ll=ll->last; while ((ll->last->btype)!=LTFT_CR && ll->last!=l) ll=ll->last; if (!ws_disable) WsPutChoices(Fs->cur_ltf,Fs->aux_ltf,l,ll); } } } } } while (!ScanChar(&sc,&ch)); ws_task=NULL; } void WsAdd(I1 *wild=TEXT_FILE_MASK,BoolU4 gloss) { WsFilemaskEntry *tempfm=AMAlloc(sizeof(WsFilemaskEntry)); if (gloss) { tempfm->next=ws_gloss_filemask_list; ws_gloss_filemask_list=tempfm; } else { tempfm->next=ws_filemask_list; ws_filemask_list=tempfm; } tempfm->mask=ANewString(wild); tempfm->flist=NULL; } void WsTraverseFilelist(BoolU4 compile,BoolU4 gloss) { WsFilemaskEntry *tempfm; U8 my_flags=(1<flist=FindFiles(tempfm->mask,&my_flags); WsAddFileList(tempfm->flist,compile,gloss); tempfm=tempfm->next; } } public void WsInit(I1 *mask) { I1 *buf,buf2[512]; I8 i,j=0; WsHashEntry *tempw; ws_disable=TRUE; //TODO:free old wordstat data ws_num_words=0; ws_num_matrix_words=0; MemSet(&ws_hash_table->body,0,sizeof(U4)*(ws_hash_table->mask+1)); ws_cur_word=NULL; ws_filemask_list=NULL; ws_gloss_filemask_list=NULL; buf=NewString(mask); do { RemoveFirstSeg(buf,";",buf2); if (*buf2) { StrCat(buf2,"/" TEXT_FILE_MASK); WsAdd(buf2,FALSE); } } while (*buf2); Free(buf); coutln "\r\n\r\nPass #1"; WsTraverseFilelist(FALSE,FALSE); ws_sorted_words=AMAlloc(PTR_SIZE*ws_num_words); for (i=0;i<=ws_hash_table->mask;i++) { tempw=ws_hash_table->body[i]; while (tempw) { ws_sorted_words[j++]=tempw; tempw=tempw->next; } } QSort(ws_sorted_words,ws_num_words,PTR_SIZE,&WsSort); for (i=0;inum=i; if (ws_num_words<=WS_MAX_MATRIX_WORDS) ws_num_matrix_words=ws_num_words; else ws_num_matrix_words=WS_MAX_MATRIX_WORDS; for (i=0;i