Hi, It has been so long since I stopped using DeleGate as a NTTP/NNTP caching proxy (or a server merging proxy) ... In these days I use the NNTP proxy feature only for NNTP/HTTP gateway in which DeleGate act as a "caching proxy" as you said. At the time when I developed NNTP proxy, the network news was very active and people were eager to check the latest articles and newsgroups without delay. Thus for example, connecting to a news server and getting NEWGROUPS at the beginning was very usual action of news-readers. Also there were several other reasons why a caching proxy for NNTP was not so necessary. And implementing cacheing servers of stateful protocol like NNTP or FTP is not so simple with underlining libraries available at that time. Since NNTP caching proxy is still interesting theme for me, I made a caching NNTP proxy which postpones connections to servers until it become necessay, based on a stab server as a threads, as the enclosed patch. It is uploaded as DeleGate/9.9.5-pre12. You can enable the feature with the NNTPCONF="ondemand" option. I tested it with two news-readers "Thunderbird" and "vin/cosmos", with the following configurations: (1) NNTPCONF=ondemand CACHE=do SERVER=nntp://www.delegate.org (2) NNTPCONF=ondemand CACHE=do SERVER=nntp MOUNT="* nntp://public.teranews.com/fj.*,comp.*" (3) NNTPCONF=ondemand CACHE=do SERVER=nntp MOUNT="* nntp://public.teranews.com/*" MOUNT="* nntp://www.delegate.org/*" These example seems working. In message <_A4550@delegate-en.ML_> on 08/20/09(20:38:22) you Florent Bautista <pyyiqbdyi-ah2v3vooc466.ml@delegate.org> wrote: |I would like to run DeleGate as a caching NNTP server. I mean, it must |have his own cache, and when it does not have the requested ressource in |its cache, it have to ask it to a "master" server. |But DeleGate does not act as I expect... | |I use this configuration: | |SERVER=nntp://login:password@giganews../ |AUTH=none |AUTHORIZER="-cmd{/var/delegate/nntp-auth.sh %U %P}" |CACHE=do |RELIABLE="*" |NNTPCONF=upact:300/150/60 |DGROOT=/var/delegate |ADMIN=newsmaster@my-usenet-service..fr | |But, every time a client connects to my caching server (DeleGate), this |one ALWAYS connects to "news-europe.giganews.com", even if my client is |not authentified nor asked a ressource !!! |The "ident" message sent to my client is the one of |"news-europe.giganews.com" ("200 News.GigaNews.Com") but not mine |(DeleGate's one). |This behaviour is not really what I expect of "a caching server"... |I used "MOUNT" option in my configuration, but it's exactly the same... | |Do you know what I mean ? And do you know how to configure DeleGate to |act as I want (i.e. DeleGate first have to look if it has the asked |ressource (article, list, etc.), and if it does not have it or if the |ressource expired, ask it to "news-europe.giganews.com" in my case) ? Cheers, Yutaka -- 9 9 Yutaka Sato <y.sato@delegate.org> http://delegate.org/y.sato/ ( ~ ) National Institute of Advanced Industrial Science and Technology _< >_ 1-1-4 Umezono, Tsukuba, Ibaraki, 305-8568 Japan Do the more with the less -- B. Fuller *** dist/src/delegate9.9.5-pre11/src/nntp.c Sun Aug 16 15:25:44 2009 --- ./src/nntp.c Wed Sep 23 18:35:45 2009 *************** *** 115,121 **** --- 115,124 ---- #define getFV(str,fld,buf) getFieldValue2(str,fld,AVStr(buf),sizeof(buf)) extern int IO_TIMEOUT; + /* #define MAXGROUPS 100000 + */ + #define MAXGROUPS 300000 #define LINESIZE 4095 #define IOBSIZE (32*1024) *************** *** 216,221 **** --- 219,228 ---- int ne_doProtoLog; int ne_lastRcode; + + const char *ne_ondemand; /* 9.9.5 NNTPCONF="ondemand" */ + char ne_redirecting_serv[64]; /* stat. for sync. in redirection */ + int ne_redirecting_tid[64]; /* tid of the exiting stab server */ } NNTPenv; static NNTPenv *NNTP_context; *************** *** 548,555 **** --- 555,594 ---- Connection *ns_Conn; int ns_serverFlags; MStr( ns_client_Addr,128); + + int ns_ondemand; /* MountOption "ondemand" */ + int ns_ondemand_tid; /* the thread for a stab server */ + int ns_ondemand_yet; /* not connected to the server yet */ + char ns_NEWGROUPS_gen; /* generated empty NEWGROUPS */ + char *ns_actcache[2][128]; /* ACTIVE list cache */ } NewsServer; + #define TRACE syslog_ERROR + typedef struct { + int f_sv[2]; + FILE *f_tcx; + FILE *f_tci; + int f_tid; + } FilterDesc; + int clearFD(FilterDesc *FD); + int setupFD(FilterDesc *FD); + int closeFD(FilterDesc *FD); + static int setOnDemand(PCStr(opts)); + static int ondemandServ(Connection *Conn,NewsServer *ns,int dmndsv[2]); + static int waitRequest(Connection *Conn,FILE *fc,FILE *tc); + static int ondemand_serv(Connection *Conn,int svsock,int clsock,NewsServer *ns); + static int waiting_ondemand_serv(FL_PAR,FILE *fs); + static int clearActCache(NewsServer *ns); + static int isServerOfX(NewsServer *ns,FILE *lf,PCStr(group)); + static int putGROUPcache(NewsServer *ns,PCStr(stat)); + static int relayLIST(FILE *tci,FILE *tc,int dofilter); + static int updateNEWGROUPS(NewsServer *ns,PCStr(qdates),int ngop); + #define NGOP_TO_UPDATE 10 /* true if to be updated for a date */ + #define NGOP_GOT_EMPTY 20 /* found no update after a date */ + #define NGOP_GOT_UPDATE 30 /* found updated after a date */ + static int hostcmpX(PCStr(h1),PCStr(h2)); + #define hostcmp(h1,h2) hostcmpX(h1,h2) + #define CACHE_ARTICLE 1 #define CACHE_OVERVIEW 2 #define CACHE_LIST 4 *************** *** 820,825 **** --- 859,868 ---- if( strcaseeq(what,"dispensable") ){ NE_dispensable = 1; }else + if( strcaseeq(what,"ondemand") ){ + /* 9.9.5 connecting target servers on demand */ + setOnDemand(value); + }else if( strcaseeq(what,"posterbase") ){ MIME_mapPosterBase = stralloc(value); }else *************** *** 882,887 **** --- 925,934 ---- } lf = LCfp(ns,LI_ACTIVE); fseek(lf,0,0); + if( NX.ne_ondemand || ns->ns_ondemand ){ + /* 9.9.5 faster ACTIVE LIST matching with cache */ + return isServerOfX(ns,lf,group); + } while( fgets(line,sizeof(line),lf) != NULL ) if( strncmp(line,group,glen) == 0 && isspace(line[glen]) ) return 1; *************** *** 1272,1277 **** --- 1319,1328 ---- FSlineCNT++; resp = fgetsTIMEOUT(BVStr(str),size,svfp); + if( resp == NULL && waiting_ondemand_serv(FL_ARG,svfp) ){ + /* 9.9.5 caused by the redirection to the real-server */ + resp = fgetsTIMEOUT(BVStr(str),size,svfp); + } if( resp == NULL ){ if( !file_isreg(fileno(svfp)) ){ /* not from cache */ server_done = 1; *************** *** 1518,1523 **** --- 1569,1578 ---- cc = ENEWS_list(tmp,ns->ns_mounted_file); else cc = LIST_uncompress(ns->ns_rfp,tmp,isactive); Verbose("LIST: got %d bytes\n",cc); + if( strcaseeq(com,"NEWGROUPS") ){ + /* 9.9.5 update "not-modifide-since" cache for NEWGROUPS */ + updateNEWGROUPS(ns,arg,cc==0?NGOP_GOT_EMPTY:NGOP_GOT_UPDATE); + } for( wx = 1; wx < nwild; wx++ ){ get_resp(ns,AVStr(line),sizeof(line)); *************** *** 1831,1836 **** --- 1886,1895 ---- int off; FILE *tmp1,*tmp2; + int wilds = 0; + FilterDesc FD; + clearFD(&FD); + tmp1 = getTmpfile("mergeLIST/1",TF_LISTTMP1,0,IOBSIZE); tmp2 = getTmpfile("mergeLIST/2",TF_LISTTMP2,0,IOBSIZE); *************** *** 1908,1913 **** --- 1967,1976 ---- if( ns->ns_isself && lastcheck == 0 ) forceup = 1; else forceup = getForceRefresh(ns,li,mtime); + if( NX.ne_ondemand && (isactive_delta && 15<=age) ){ + /* 9.9.5 for vin/cosmos to be age<15 */ + age = 0; + } if( ns->ns_nocache & CACHE_LIST ){ /* but LIST cache is necessary for isServerOf */ *************** *** 1936,1944 **** --- 1999,2011 ---- if( isactive_wildmat && 0 < ns->ns_withLISTwildmat ){ Xsscanf(arg,"%*s %[^\r\n]",AVStr(wm)); wildmats[si] = sendWildmats(ns,"LIST ACTIVE",wm); + wilds++; }else if( isactive && 0 < ns->ns_withLISTwildmat && getLISTwildmat(ns,AVStr(wm)) ) + { wildmats[si] = sendWildmats(ns,"LIST ACTIVE",wm); + wilds++; + } else FPUTS(req,ns->ns_wfp); } *************** *** 1978,1983 **** --- 2045,2062 ---- } } + if( NX.ne_ondemand ){ + /* 9.9.5 on-the-fly streaming of ACTIVE LISTs to the client */ + if( 0 < wilds ){ + /* wild. filter is not implemented in relayLIST */ + }else{ + setupFD(&FD); + FD.f_tid = thread_fork(0x100000,0,"ActLIST", + (IFUNCP)relayLIST,FD.f_tci,tc,dofilter); + fputs(msg,FD.f_tcx); + fflush(FD.f_tcx); + } + } fseek(tmp1,0,0); for( si = 0; si < nserv; si++ ){ FILE *cachefp; *************** *** 1993,1998 **** --- 2072,2093 ---- if( cachable && !strcaseeq(ns->ns_proto,"pop") ) fseek(cachefp,0,0); + if( FD.f_tcx ){ + if( reuseit[si] ){ + if( fatal == 0 ){ + getLISTcache(ns,si,li,cachefp,FD.f_tcx); + } + }else{ + if( cachable ) /* switch the cache file for LIST */ + fprintf(FD.f_tcx,"#>CACHE %s\r\n",LCpath(ns,li)); + recvLISTs(ns,com,arg,FD.f_tcx,isactive,wildmats[si]); + if( cachable ) + fprintf(FD.f_tcx,"#>CACHE .\r\n"); + LCorigmtime(ns,li) = 0; + clearActCache(ns); + } + continue; + } if( reuseit[si] ){ if( fatal != 0 || tc == NULLFP() ) continue; *************** *** 2023,2028 **** --- 2118,2128 ---- if( cachable ) putLISTcache(ns,si,li,tocache,cachefp,private_cache); } + if( FD.f_tcx ){ + Fputs(".\r\n",FD.f_tcx); + closeFD(&FD); + return; + } if( tc == NULLFP() ) return; *************** *** 2229,2234 **** --- 2329,2340 ---- goto ADD; } } + if( ns->ns_ondemand_yet ){ + /* 9.9.5 don't initiate connection to server */ + sv1log("==== don't try PATHHOST in on-demand %s:%d\n", + ns->ns_host,ns->ns_port); + return; + } if( get_pathhost0(ns,AVStr(pathhost),"junk") == 0 ) if( get_pathhost0(ns,AVStr(pathhost),"control") == 0 ) *************** *** 2282,2287 **** --- 2388,2396 ---- if( resp[0] && age < UPCONF ){ sv1log("reuse LIST [wildmat][age=%d] %s",age,resp); rcode = atoi(resp); + if( rcode == 400 || rcode == 401 ){ + rcode = 0; /* 9.9.5 ignore temp. error */ + } } } *************** *** 3454,3459 **** --- 3563,3571 ---- sv1log("## S-C %s",msg); INNsetup(ns,AVStr(msg)); + if( ns->ns_ondemand_yet ){ + /* 9.9.5 don't cache msg. generated by a stab server */ + }else put_cache(ns,"NNTP-open","lib/opening",msg); if( ns->ns_rw == 0 ){ *************** *** 3550,3555 **** --- 3662,3672 ---- lastRcode = code; if( code == 200 || code == 201 ){ set_opening(ns,AVStr(line)); + if( ns->ns_ondemand_yet ){ + /* 9.9.5 don't initiate connection to server */ + sv1log("==== don't DATE for on-demand %s:%d\n", + ns->ns_host,ns->ns_port); + }else if( 1 < nservers_remote() ){ CStr(resp,256); fprintf(ns->ns_wfp,"DATE\r\n"); *************** *** 3557,3564 **** --- 3674,3688 ---- fgetsFS(AVStr(resp),sizeof(resp),fs); sv1log("#DATE> %s",resp); } + if( NX.ne_ondemand && strcaseeq(com,"MODE") ){ + /* 9.9.5 should put it into MODE-cache */ + } if( nservers_remote() != 1 ) return 0; + if( NX.ne_ondemand && !strcaseeq(com,"MODE") ){ + /* 9.9.5 greeting was put at the beginning */ + return 0; + } } else if( NE_dispensable ){ *************** *** 3617,3622 **** --- 3741,3750 ---- CStr(rgroup,1024); CStr(remain,1024); + if( NX.ne_ondemand && !ns->ns_ondemand_yet ){ + /* 9.9.5 put into the GROUP cache */ + putGROUPcache(ns,line); + } rgroup[0] = ngroup[0] = remain[0] = 0; Xsscanf(line,"%*d %d %d %d %[^ \r\n]%[^\r\n]",&num,&min,&max,AVStr(rgroup),AVStr(remain)); if( rgroup[0] ){ *************** *** 4451,4456 **** --- 4579,4588 ---- } elapsed = time(0) - start; + if( nready < 0 && waiting_ondemand_serv(FL_ARG,0) ){ + /* 9.9.5 redirected to the real-server */ + continue; + } if( nready < 0 || nready == 0 && elapsed == 0 ){ sv1log("EXIT: NO ready stream. %d/%d\n", nready,elapsed); *************** *** 5787,5792 **** --- 5919,5926 ---- ns->ns_mounted_file++; if( si == NserverN ){ + int dmndsv[2] = {-1,-1}; + set_realserver(Conn,"nntp",host,port); if( isMYSELF(host) ){ FromS = ToS = -1; *************** *** 5797,5802 **** --- 5931,5943 ---- if( strcmp(opts,"delay") == 0 ){ FromS = ToS = -1; opts = ""; + }else + if( NX.ne_ondemand || isinListX(opts,"ondemand","ch") ){ + /* 9.9.5 init. with a socketpair to a stab server */ + Socketpair(dmndsv); + FromS = ToS = dmndsv[1]; + strcpy(ns->ns_proto,proto); + ns->ns_ondemand = 1; }else{ strcpy(ns->ns_proto,proto); if( connect_serv(Conn,ns) < 0 ) *************** *** 5805,5810 **** --- 5946,5955 ---- nsid = addServer1(Conn,proto,user,pass,host,port,FromC,ToC,FromS,ToS); ns = toNS(nsid); + if( 0 <= dmndsv[0] ){ + /* 9.9.5 insert a stab server as a thread */ + ondemandServ(Conn,ns,dmndsv); + } } else{ /* SERVER=nntp://user:pass@server/ */ *************** *** 6012,6018 **** --- 6157,6176 ---- } compressLIST = 0; + if( NX.ne_ondemand ){ + /* 9.9.5 let servers connected on demand */ + putIdent(tc,""); + fflush(tc); + if( waitRequest(Conn,fc,tc) != 0 ){ + fcloseFILE(tc); + fclose(fc); + return; + } + } if( NserverN == 0 ){ + if( NX.ne_ondemand ){ + /* 9.9.5 no imm. connect_serv() for SERVER=nntp://serv */ + }else if( addCurrent(Conn) < 0 ) goto rejected; *************** *** 6024,6033 **** --- 6182,6194 ---- }else{ QueueLeng = 0; if( nservers_remote() == 1 ){ int s1 = server1(); + if( NX.ne_ondemand == 0 ) fputs(Nservers[s1].ns_openingmsg,tc); fflush(tc); } } + if( NX.ne_ondemand ){ + }else if( nservers_remote() != 1 ) putIdent(tc,""); *************** *** 7761,7763 **** --- 7922,8647 ---- } return closed; } + + /* + * 9.9.5 on-demand server + */ + static CriticalSec connCSC; + static int setOnDemand(PCStr(opts)){ + NX.ne_ondemand = stralloc(opts); + return 0; + } + int clearFD(FilterDesc *FD){ + bzero(FD,sizeof(FilterDesc)); + return 0; + } + int setupFD(FilterDesc *FD){ + pipe(FD->f_sv); + FD->f_tci = fdopen(FD->f_sv[0],"r"); + FD->f_tcx = fdopen(FD->f_sv[1],"w"); + return 0; + } + int closeFD(FilterDesc *FD){ + fclose(FD->f_tcx); + thread_wait(FD->f_tid,3*1000); + fclose(FD->f_tci); + return 0; + } + /* real hostcmp() could be too heavy */ + static int hostcmpX(PCStr(h1),PCStr(h2)){ + if( NX.ne_ondemand ){ + if( strcasecmp(h1,h2) == 0 ){ + return 0; + } + return -1; + }else{ + return hostcmp(h1,h2); + } + } + static int clearActCache1(NewsServer *ns,int neg){ + int ai; + const char *act1; + int nclr = 0; + + for( ai = 0; ai < elnumof(ns->ns_actcache[0]); ai++ ){ + if( act1 = ns->ns_actcache[neg][ai] ){ + free((char*)act1); + ns->ns_actcache[neg][ai] = 0; + nclr++; + } + } + return nclr; + } + static int clearActCache(NewsServer *ns){ + int nclr = 0; + nclr += clearActCache1(ns,0); + nclr += clearActCache1(ns,1); + if( nclr ){ + TRACE("---- cleared active cache (%d)\n",nclr); + } + return 0; + } + #define IsDelim(ch) (isspace(ch) || ch == 0) + static int putActCache(PCStr(wh),NewsServer *ns,PCStr(grp),PCStr(act),int neg){ + int glen = strlen(grp); + const char *act1; + IStr(actb,256); + int ai; + int asiz; + int ni = ns->ns_nsid; + + if( neg ) neg = 1; + lineScan(act,actb); + for( ai = 0; ai < elnumof(ns->ns_actcache[neg]); ai++ ){ + act1 = ns->ns_actcache[neg][ai]; + if( act1 == 0 ){ + ns->ns_actcache[neg][ai] = stralloc(actb); + TRACE("--AC %s[%d] ADD[%d] %s\n",wh,ni,ai,actb); + return 2; + } + if( strncmp(act1,grp,glen) == 0 && IsDelim(act1[glen]) ){ + TRACE("--AC %s[%d] DUP[%d] %s\n",wh,ni,ai,actb); + return 1; + } + } + return 0; + } + static int getActCache(PCStr(wh),NewsServer *ns,PCStr(grp),PVStr(act),int neg){ + int glen = strlen(grp); + const char *act1; + int ai; + int ni = ns->ns_nsid; + + if( neg ) neg = 1; + for( ai = 0; ai < elnumof(ns->ns_actcache[neg]); ai++ ){ + act1 = ns->ns_actcache[neg][ai]; + if( act1 == 0 ){ + continue; + } + if( strncmp(act1,grp,glen) == 0 && IsDelim(act1[glen]) ){ + sprintf(act,"%s\r\n",act1); + if( neg ) + TRACE("--AC %s/%d NEG[%d] %s\n",wh,ni,ai,act1); + else TRACE("--AC %s/%d HIT[%d] %s\n",wh,ni,ai,act1); + return 1; + } + } + return 0; + } + static int getGROUPcache(NewsServer *ns,PCStr(group),PVStr(stat)); + static int isServerOfX(NewsServer *ns,FILE *lf,PCStr(grp)){ + IStr(act1,1024); + double St = Time(); + int glen = strlen(grp); + int siz = 0; + + /* should use Hsearch() or shared-memory for a large list */ + if( getActCache("isServer",ns,grp,AVStr(act1),0) ){ + return 2; + } + if( getActCache("isServer",ns,grp,AVStr(act1),1) ){ + return 0; + } + if( getGROUPcache(ns,grp,AVStr(act1)) ){ + if( act1[0] == '2' ){ + TRACE("--GC HIT[%d] %s",ns->ns_nsid,act1); + return 1; + } + } + while( fgets(act1,sizeof(act1),lf) != NULL ){ + siz += strlen(act1); + if( act1[0] == grp[0] ) + if( strncmp(act1,grp,glen) == 0 && isspace(act1[glen]) ){ + TRACE("## FOUND[%d] %.2f %s\n",ns->ns_nsid,Time()-St, + grp); + putActCache("isServerOf",ns,grp,act1,0); + return 1; + } + } + putActCache("isServerOf",ns,grp,grp,1); + TRACE("## NOT FOUND[%d] %.2f '%s'\n",ns->ns_nsid,Time()-St,grp); + return 0; + } + static int relayLIST(FILE *tci,FILE *tc,int dofilter){ + IStr(line,1024); + int li; + int size = 0; + double St = Time(); + double PSt; + FILE *cfp = 0; + IStr(path,256); + IStr(cpath,256); + + PSt = St; + for( li = 0; ; li++ ){ + if( fgets(line,sizeof(line),tci) == NULL ){ + break; + } + if( line[0] == '#' && line[1] == '>' ){ + TRACE(">>>> %s",line); + clearVStr(path); + Xsscanf(line,"#>CACHE %[^\r\n]",AVStr(path)); + if( path[0] ){ + if( cfp ){ + TRACE(">>>> done=%d %s\n",ftell(cfp), + cpath); + fclose(cfp); + cfp = 0; + } + if( streq(path,".") ){ + }else{ + strcpy(cpath,path); + cfp = fopen(cpath,"r+"); + if( cfp == 0 ){ + cfp = fopen(cpath,"w"); + } + TRACE(">>>> start %X %s\n",cfp,cpath); + } + } + continue; + } + /* should do filter_active() before cache & relay */ + + if( cfp ){ + fputs(line,cfp); + } + fputs(line,tc); + size += strlen(line); + if( (li % 2000) == 0 && 2 < (Time()-PSt) ){ + TRACE(">>>> LIST %d %d %.1fK/s (%.2f)\n",li, + size,(size/(Time()-St))*0.001,Time()-St); + PSt = Time(); + } + } + if( cfp ){ + TRACE(">>>> done=%d %s\n",ftell(cfp),cpath); + fclose(cfp); + } + TRACE(">>>> LIST %d %d (%.2f) DONE\n",li,size,Time()-PSt); + return 0; + } + /* the "not-modified-since" cache of NEWGROUPS to suppress connection + * to the server initiated by NEWGROUPS. + */ + static int updateNEWGROUPS(NewsServer *ns,PCStr(qdates),int ngop){ + int qdate; + FILE *dfp; + IStr(path,1024); + IStr(cdates,1024); + int age = -1; + int cdate = -1; /* known "not-modified-since" of NEWGROUPS */ + int rcode = -1; + int Lexpire = UPACT_SOME; + int update; /* to be updated for TO_UPDATE and GOT_EMPTY */ + + cache_path("nntp",ns->ns_host,ns->ns_port,"NEWGROUPS-date",AVStr(path)); + if( dfp = dirfopen("NG",AVStr(path),"r+") ){ + age = file_age(path,dfp); + Fgets(AVStr(cdates),sizeof(cdates),dfp); + cdate = YMD_HMS_toi(cdates); + }else{ + dfp = dirfopen("NG",AVStr(path),"w+"); + } + if( dfp == NULL ){ + return -1; + } + fseek(dfp,0,0); + qdate = YMD_HMS_toi(qdates); + update = 0; + switch( ngop ){ + case NGOP_TO_UPDATE: + /* inquired date older than the known "not-modified-since" */ + if( cdate == -1 ) update = 11; else + if( qdate < cdate ) update = 12; else + if( Lexpire < age ) update = 13; + rcode = update; + break; + case NGOP_GOT_EMPTY: + if( ns->ns_NEWGROUPS_gen ){ + /* ignore the generated resp. from self */ + }else + if( cdate == -1 ) update = 21; else + if( qdate < cdate ) update = 22; else + if( Lexpire < age ) update = 23; + if( update ){ + TRACE("## NGRP empty Q-%X C=%X\n",qdate,cdate); + fprintf(dfp,"%s\r\n",qdates); + Ftruncate(dfp,0,1); + } + break; + case NGOP_GOT_UPDATE: + if( cdate < qdate ){ + /* "not-modified-since" has become unknown */ + TRACE("## NGRP updated Q-%X C=%X\n",qdate,cdate); + Ftruncate(dfp,0,0); + update = 30; + } + break; + } + TRACE("## NGRP %d %d age=%d update=%d Q[%s]%X C[%s]%X %s\n",ngop, + ns->ns_NEWGROUPS_gen,age,update,qdates,qdate,cdates,cdate,path); + fflush(dfp); + return rcode; + } + static int waiting_ondemand_serv(FL_PAR,FILE *fs){ + int wi; + int terr = -2,tid; + int fx; + + if( fs && file_isreg(fileno(fs)) ){ + return 0; + } + fx = fileno(fs) % elnumof(NX.ne_redirecting_serv); + TRACE("==== waiting_ondemand[%d] %s:%d (%d) EOF=%d\n",fx, + FL_BAR,NX.ne_redirecting_serv[fx],fs?feof(fs):0); + if( NX.ne_redirecting_serv[fx] == 0 ){ + return 0; + } + for( wi = 0; wi < 50; wi++ ){ + if( NX.ne_redirecting_serv[fx] == 2 ){ + if( fs ){ + clearerr(fs); + } + if( tid = NX.ne_redirecting_tid[fx] ){ + terr = thread_wait(tid,500); + NX.ne_redirecting_tid[fx] = 0; + } + TRACE("==== waiting_ondemand %s:%d (%d) OK %X/%d\n", + FL_BAR,NX.ne_redirecting_serv[fx],tid,terr); + NX.ne_redirecting_serv[fx] = 0; + return 1; + } + msleep(100); + } + TRACE("==== waiting_ondemand %s:%d ERR sync (%d)\n",FL_BAR, + NX.ne_redirecting_serv[fx]); + return 1; + } + static int putGROUPbyLIST(NewsServer *ns,FILE *lfp,PCStr(group),FILE *tc){ + IStr(line,256); + int len; + int age = file_age(group,lfp); + double St = Time(); + int li; + int min,max,total; + + TRACE("==== [%d] LIST age=%d %s\n",ns->ns_nsid,age,group); + len = strlen(group); + if( getActCache("getGL",ns,group,AVStr(line),0) ){ + goto FOUND; + } + if( getActCache("getGL",ns,group,AVStr(line),1) ){ + return 0; + } + for( li = 0; fgets(line,sizeof(line),lfp) != NULL; li++ ){ + if( line[0] == group[0] ) + if( strncmp(line,group,len) == 0 ){ + TRACE("==== [%d] gc GOT (%d %.2f) %s",ns->ns_nsid, + li,Time()-St,line); + putActCache("getGL",ns,group,line,0); + goto FOUND; + } + } + TRACE("==== [%d] not in LIST (%d %.2f) age=%d %s\n", + ns->ns_nsid,li,Time()-St,age,group); + putActCache("getGL",ns,group,group,1); + return 0; + FOUND: + min = max = total = -1; + sscanf(line,"%*s %d %d",&max,&min); + if( max != 0 ) + total = max - min + 1; + else total = 0; + if( 0 < max ){ + TRACE("==== GROUP 211 %d %d %d %s\n",total,min,max,group); + fprintf(tc,"211 %d %d %d %s\r\n",total,min,max,group); + return 1; + } + return 0; + } + /* forward the current status to the real server */ + static void resume0(NewsServer *ns,FILE *ts,FILE *fs){ + IStr(resp,256); + const char *group = ns->ns_curgroupR?ns->ns_curgroupR:""; + int anum = ns->ns_curanum; + + TRACE("==== curgroup[%s:%d]\n",group,anum); + if( *group ){ + fprintf(ts,"GROUP %s\r\n",group); + fflush(ts); + fgets(resp,sizeof(resp),fs); + if( resp[0] == 0 ) strcpy(resp,"\n"); + TRACE("==== [%s:%d] rdy=%d %s",group,anum,ready_cc(fs),resp); + } + if( anum ){ + fprintf(ts,"STAT %d\r\n",anum); + fflush(ts); + fgets(resp,sizeof(resp),fs); + if( resp[0] == 0 ) strcpy(resp,"\n"); + TRACE("==== [%s:%d] rdy=%d %s",group,anum,ready_cc(fs),resp); + } + } + static int getStatusCache(NewsServer *ns,PCStr(gpath),PVStr(stat)){ + IStr(cpath,256); + IStr(sta1,256); + refQStr(sp,sta1); + FILE *gfp; + int code; + int age; + + cache_path("nntp",ns->ns_host,ns->ns_port,gpath,AVStr(cpath)); + if( gfp = dirfopen("StatusCache",AVStr(cpath),"r") ){ + age = file_age(cpath,gfp); + fgets(sta1,sizeof(sta1),gfp); + if( sp = strpbrk(sta1,"\r\n") ) + clearVStr(sp); + fclose(gfp); + code = atoi(sta1); + sprintf(stat,"%s\r\n",sta1); + if( UPACT_SOME < age ){ + TRACE("---- exp cache age=%d: %s",age,stat); + return 0; + } + if( 100 <= code && code < 999 ){ + TRACE("---- got cache age=%d: %s",age,stat); + return 1; + } + } + return 0; + } + static int toGpath(PCStr(base),PCStr(group),int ncol,PVStr(gpath)){ + IStr(md5,64); + toMD5(group,md5); + setVStrEnd(md5,ncol); + sprintf(gpath,"%s/=%s/%s",base,md5,group); + return 0; + } + static int getGROUPcache(NewsServer *ns,PCStr(group),PVStr(stat)){ + IStr(gpath,256); + toGpath("GROUP",group,2,AVStr(gpath)); + return getStatusCache(ns,gpath,BVStr(stat)); + } + static int putGROUPcache(NewsServer *ns,PCStr(stat)){ + IStr(group,256); + IStr(gpath,256); + IStr(cpath,256); + FILE *gfp; + + Xsscanf(stat,"%*d %*d %*d %*d %s",AVStr(group)); + toGpath("GROUP",group,2,AVStr(gpath)); + cache_path("nntp",ns->ns_host,ns->ns_port,gpath,AVStr(cpath)); + if( gfp = dirfopen("GROUPcache",AVStr(cpath),"w") ){ + TRACE("---- put cache: %s",stat); + fprintf(gfp,"%s",stat); + fclose(gfp); + return 0; + } + return -1; + } + static int putActLIST(NewsServer *ns,PCStr(arg),FILE *tc){ + FILE *lfp; + + if( *arg && !strcaseeq(arg,"ACTIVE") ){ + return 0; + } + lfp = LCfp(ns,LI_ACTIVE); + if( lfp == 0 ){ + return 0; + } + TRACE("==== [%d] LIST[%s] age=%d\n",ns->ns_nsid,arg,file_age(arg,lfp)); + fprintf(tc,"215 Newsgroups in form \"group high low flags\".\r\n"); + fseek(lfp,0,0); + copyfile1(lfp,tc); + fputs(".\r\n",tc); + return 1; + } + static int putArtSTAT(NewsServer *ns,PCStr(arg),FILE *tc){ + FILE *afp; + IStr(path,256); + IStr(msgid,256); + int anum = atoi(arg); + + afp = NNTP_openARTICLE(ns->ns_nsid,-1, + ns->ns_curgroupR,anum,AVStr(path)); + if( afp == 0 ){ + return 0; + } + fgetsHeaderField(afp,"Message-ID",AVStr(msgid),sizeof(msgid)); + fclose(afp); + fprintf(tc,"223 %d %s\r\n",anum,msgid); + return 1; + } + #define Get_Cache(host,port,path,msg) \ + get_cache("NNTP-cache",host,port,path,AVStr(msg),sizeof(msg)) + int ShutdownSocket(int); + static int inconnect; + static int connect_servCSC(Connection *ConnX,NewsServer *ns){ + int mok; + int ntry = 0; + int toS; + double St = Time(); + Connection ConnBuf,*Conn = &ConnBuf; + + ConnBuf = *ConnX; + mok = enterCSCX(connCSC,10*1000); + while( 0 < inconnect ){ /* maybe running on a OS without mutex */ + msleep(300); + if( 30 < Time()-St ){ + TRACE("==== #### TIMEOUT waiting connect(%d)...[%X]\n", + inconnect,PRTID(ns->ns_nsid)); + return -1; + } + ntry++; + } + inconnect++; + strcpy(REAL_PROTO,ns->ns_proto); + strcpy(REAL_HOST,ns->ns_host); + REAL_PORT = ns->ns_port; + toS = connect_serv(Conn,ns); + inconnect--; + leaveCSC(connCSC); + TRACE("==== connected [%d] mutex=%d,%d (%.2f) => %s:%d\n", + toS,mok,ntry,Time()-St,ns->ns_host,ns->ns_port); + return toS; + } + static int ondemand_serv(Connection *Conn,int svsock,int clsock,NewsServer *ns){ + FILE *tc,*fc; + IStr(msg,256); + IStr(req,256); + IStr(com,256); + IStr(arg,256); + const char *dp; + const char *host = ns->ns_host; + int port = ns->ns_port; + int nsvsock; + IStr(path,1024); + int didputopen = 0; + FILE *lfp; + int Lexpire = UPACT_SOME; + int fx = clsock % elnumof(NX.ne_redirecting_serv); + + ns->ns_ondemand_yet = 1; + fc = fdopen(svsock,"r"); + tc = fdopen(svsock,"w"); + TRACE("==== stab #### [%d] SERVER=nntp://%s:%d\n", + ns->ns_nsid,host,port); + if( Get_Cache(host,port,"lib/opening",msg) ){ + didputopen = 1; + fprintf(tc,"%s",msg); + }else{ + goto DO_CONNECT; + } + for(;;){ + fflush(tc); + if( fgets(req,sizeof(req),fc) == 0 ){ + TRACE("==== stab Q: EOS\n"); + break; + } + TRACE("==== stab [%d] Q: [%s] %s", + ns->ns_nsid,ns->ns_curgroupR?ns->ns_curgroupR:"",req); + dp = wordScan(req,com); + lineScan(dp,arg); + + if( strcaseeq(com,"MODE") ){ + if( strcaseeq(arg,"READER") ){ + if( 0 ){ + /* if cached in MODE-cache */ + }else{ + fprintf(tc,"200 ==== enabled\r\n"); + } + continue; + } + } + if( strcaseeq(com,"HELP") ){ + if( Get_Cache(host,port,"lib/HELP",msg) ){ + fprintf(tc,"%s\r\n.\r\n",msg); + continue; + }else{ + break; + } + } + if( strcaseeq(com,"DATE") ){ + CStr(sdate,32); + StrftimeGMT(AVStr(sdate),sizeof(sdate),"%Y%m%d%H%M%S", + time(0),0); + fprintf(tc,"111 %s\r\n",sdate); + continue; + } + if( strcaseeq(com,"NEWGROUPS") ){ + if( updateNEWGROUPS(ns,arg,NGOP_TO_UPDATE) ){ + ns->ns_NEWGROUPS_gen = 0; + break; + }else{ + ns->ns_NEWGROUPS_gen = 1; + fprintf(tc,"231 not-modified.\r\n"); + fprintf(tc,".\r\n"); + continue; + } + } + if( strcaseeq(com,"STAT") ){ + if( putArtSTAT(ns,arg,tc) ){ + continue; + } + } + if( strcaseeq(com,"LIST") ){ + /* + if( putActLIST(ns,arg,tc) ){ + continue; + } + */ + } + if( strcaseeq(com,"GROUP") ){ + if( getGROUPcache(ns,arg,AVStr(msg)) ){ + fputs(msg,tc); + continue; + } + lfp = LCfp(ns,LI_ACTIVE); + if( lfp ){ + int age; + if( Lexpire < (age=file_age("LIST",lfp)) ){ + TRACE("==== LIST-cache age=%d > %d\n", + age,Lexpire); + }else{ + fseek(lfp,0,0); + if( putGROUPbyLIST(ns,lfp,arg,tc) ){ + continue; + } + } + }else{ + TRACE("==== GROUP %s not-in-LIST-cache\n",arg); + } + if( streq(arg,"junk") || streq(arg,"control") ){ + sprintf(msg,"500 ign. %u GROUP %s",time(0),arg); + TRACE("%s\n",msg); + fprintf(tc,"%s\r\n",msg); + continue; + } + } + break; + } + + DO_CONNECT: + ns->ns_ondemand_yet = 0; + if( feof(fc) || !IsAlive(ClientSock) ){ + goto EXIT; + } + TRACE("==== stab #### [%d] CONNECTING to '%s:%d' for: %s%s", + ns->ns_nsid,ns->ns_host,ns->ns_port,req, + strchr(req,'\n')?"":"\n"); + + if( (nsvsock = connect_servCSC(Conn,ns)) < 0 ){ + TRACE("==== stab ERROR, connection failed.\n"); + fprintf(tc,"400 ondemand-connect failed.\r\n"); + }else{ + int tmpsv; + FILE *fs,*ts; + + tmpsv = dup(nsvsock); + fs = fdopen(tmpsv,"r"); + TRACE("==== sockets [%d %d] [%d %d] [%d] %X\n", + svsock,clsock, nsvsock,tmpsv,ToSX,fs); + if( fs == NULL ){ + TRACE("==== #### fdopen[%d][%d]%X\n",nsvsock,tmpsv,fs); + _Finish(-1); + } + setbuffer(fs,0,0); + ts = fdopen(tmpsv,"w"); + setbuffer(ts,0,0); /* for FC6 and Deb3 */ + /* should do fPollIns(fc,fs) here */ + fgets(msg,sizeof(msg),fs); + TRACE("==== stab server SAYS: %s",msg); + if( msg[0] == '2' ){ + put_cache(ns,"NNTP-open","lib/opening",msg); + } + if( didputopen == 0 || msg[0] != '2' ){ + fputs(msg,tc); + fflush(tc); + if( msg[0] != '2' ){ + TRACE("#### real server says: %s",msg); + goto EXIT; + } + } + resume0(ns,ts,fs); + if( req[0] ){ + fputs(req,ts); + TRACE("==== forw-1 %s",req); + } + while( 0 < fPollIn(fc,1000) ){ /* pipelined requests */ + if( fgets(req,sizeof(req),fc) ){ + fputs(req,ts); + TRACE("==== forw-2 %s",req); + }else{ + break; + } + } + fcloseFILE(ts); + fclose(fs); + + NX.ne_redirecting_tid[fx] = ns->ns_ondemand_tid; + NX.ne_redirecting_serv[fx] = 1; + ShutdownSocket(svsock); + /* should wait receiver to exit from select(clsock) or + * recv(clsock) and start waiting_ondemand_serv() ... + */ + msleep(100); + dup2(nsvsock,clsock); /* might interrupt select(clsock) */ + close(nsvsock); + TRACE("==== stab OK, connected.\n"); + } + EXIT: + TRACE("==== stab done %X\n",TID); + fcloseFILE(tc); + fclose(fc); + NX.ne_redirecting_serv[fx] = 2; + return 0; + } + static int ondemandServ(Connection *Conn,NewsServer *ns,int dmndsv[2]){ + int tid; + + setupCSC("ActLIST",connCSC,sizeof(connCSC)); + ns->ns_islocal = 0; + tid = thread_fork(0x100000,0,"NNTP-on-demand",(IFUNCP)ondemand_serv, + Conn,dmndsv[0],dmndsv[1],ns); + TRACE("==== stab SERVER=nntp://%s:%d [%X]\n", + ns->ns_host,ns->ns_port,tid); + ns->ns_ondemand_tid = tid; + return 0; + } + /* Thunderbird makes connections without sending requests other than "QUIT" */ + int recvPEEK(int sock,PVStr(buf),int size); + static int waitRequest(Connection *Conn,FILE *fc,FILE *tc){ + double Start = Time(); + IStr(com,128); + IStr(mod,128); + IStr(req,128); + int nready; + int rcc; + + if( (nready = fPollIn(fc,3*1000)) == 0 ){ + nready = fPollIn(fc,30*1000); + } + if( 0 < nready ){ + rcc = recvPEEK(FromC,AVStr(req),sizeof(req)-1); + if( rcc <= 0 ){ + nready = -2; + }else + if( 0 < rcc ){ + setVStrEnd(req,rcc); + lineScan(req,com); + if( strcaseeq(com,"QUIT") ){ + fprintf(tc,"205\r\n"); + nready = -3; + } + if( strcaseeq(com,"MODE READER") ){ + TRACE("==== %s\n",com); + nready = 3; + } + } + } + if( nready <= 0 ){ + TRACE("---- reset without request (%d) %.2f [%s][%s]\n", + nready,Time()-Start,mod,com); + return 1; + } + return 0; + }