diff -r 88ea8de5bdf7 sys/include/venti.h --- a/sys/include/venti.h Sat May 26 00:00:00 2012 +0200 +++ b/sys/include/venti.h Tue May 29 00:00:00 2012 +0200 @@ -83,7 +83,6 @@ { VtScoreSize = 20, VtMaxStringSize = 1024, - VtMaxLumpSize = 56*1024, VtPointerDepth = 7 }; #define VtMaxFileSize ((1ULL<<48)-1) @@ -134,7 +133,9 @@ _VtEntryDir = 1<<1, /* a directory */ _VtEntryDepthShift = 2, /* shift for pointer depth */ _VtEntryDepthMask = 7<<2, /* mask for pointer depth */ - VtEntryLocal = 1<<5 /* for local storage only */ + VtEntryLocal = 1<<5, /* for local storage only */ + _VtEntryBig = 1<<6, + VtEntryNoArchive = 1<<7, /* for local storage only */ }; enum { @@ -143,8 +144,8 @@ struct VtEntry { ulong gen; /* generation number */ - ushort psize; /* pointer block size */ - ushort dsize; /* data block size */ + ulong psize; /* pointer block size */ + ulong dsize; /* data block size */ uchar type; uchar flags; uvlong size; @@ -159,14 +160,15 @@ char name[128]; char type[128]; uchar score[VtScoreSize]; /* to a Dir block */ - ushort blocksize; /* maximum block size */ + ulong blocksize; /* maximum block size */ uchar prev[VtScoreSize]; /* last root block */ }; enum { VtRootSize = 300, - VtRootVersion = 2 + VtRootVersion = 2, + _VtRootVersionBig = 1<<15, }; void vtrootpack(VtRoot*, uchar*); @@ -293,7 +295,7 @@ uint nauth; /* TauthX, RauthX */ uchar score[VtScoreSize]; /* Tread, Rwrite */ uchar blocktype; /* Tread, Twrite */ - ushort count; /* Tread */ + uint count; /* Tread */ Packet *data; /* Rread, Twrite */ }; @@ -334,7 +336,9 @@ }; VtConn* vtconn(int infd, int outfd); +int vtreconn(VtConn*, int, int); VtConn* vtdial(char*); +int vtredial(VtConn*, char *); void vtfreeconn(VtConn*); int vtsend(VtConn*, Packet*); Packet* vtrecv(VtConn*); @@ -378,6 +382,10 @@ int vtsync(VtConn*); int vtping(VtConn*); +/* sha1 */ +void vtsha1(uchar score[VtScoreSize], uchar*, int); +int vtsha1check(uchar score[VtScoreSize], uchar*, int); + /* * Data blocks and block cache. */ @@ -397,7 +405,8 @@ uchar *data; uchar score[VtScoreSize]; - uchar type; /* BtXXX */ + uchar type; /* VtXXX */ + ulong size; /* internal to cache */ int nlock; @@ -415,15 +424,14 @@ u32int vtglobaltolocal(uchar[VtScoreSize]); void vtlocaltoglobal(u32int, uchar[VtScoreSize]); -VtCache*vtcachealloc(VtConn*, int blocksize, ulong nblocks); +VtCache*vtcachealloc(VtConn*, ulong maxmem); void vtcachefree(VtCache*); VtBlock*vtcachelocal(VtCache*, u32int addr, int type); -VtBlock*vtcacheglobal(VtCache*, uchar[VtScoreSize], int type); -VtBlock*vtcacheallocblock(VtCache*, int type); +VtBlock*vtcacheglobal(VtCache*, uchar[VtScoreSize], int type, ulong size); +VtBlock*vtcacheallocblock(VtCache*, int type, ulong size); void vtcachesetwrite(VtCache*, int(*)(VtConn*, uchar[VtScoreSize], uint, uchar*, int)); void vtblockput(VtBlock*); -u32int vtcacheblocksize(VtCache*); int vtblockwrite(VtBlock*); VtBlock*vtblockcopy(VtBlock*); void vtblockduplock(VtBlock*); @@ -442,6 +450,7 @@ int local; VtBlock *b; /* block containing this file */ uchar score[VtScoreSize]; /* score of block containing this file */ + int bsize; /* size of block */ /* immutable */ VtCache *c; diff -r 88ea8de5bdf7 sys/man/1/vac --- a/sys/man/1/vac Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/1/vac Tue May 29 00:00:00 2012 +0200 @@ -6,6 +6,9 @@ [ .B -mqsv ] [ +.B -a +.I vacfile +] [ .B -b .I blocksize ] [ @@ -23,12 +26,15 @@ ] [ .B -h .I host +] [ +.B -x +.I excludefile ] .I file ... .PP .B unvac [ -.B -Tctv +.B -Tcdtv ] [ .B -h .I host @@ -64,11 +70,27 @@ vac:64daefaecc4df4b5cb48a368b361ef56012a4f46 .EE .PP -The options to -.I vac -are: -.TF "-d\fI oldvacfile" -.PD +The options are: +.TP +.BI -a " vacfile +Specifies that vac should create or update a backup archive, inserting +the files under an extra two levels of directory hierarchy named +.I yyyy/mmdd +(year, month, day) +in the style of the dump file system +(see Plan 9's \fIfs\fR(4)). +If +.I vacfile +already exists, an additional backup day is added to the +existing hierarchy, behaving as though the +.B -d +flag was specified giving the most recent backup tree in the archive. +Typically, this option +is used as part of a nightly backup script. +This option cannot be used with +.B -d +or +.BR -f . .TP .BI -b " blocksize Specifies the block size that data will be broken into. @@ -90,6 +112,12 @@ Do not include the file or directory specified by .IR exclude . This option may be repeated multiple times. +.I Exclude +can be a shell pattern as accepted by +.IR rc (1), +with one extension: +.B \&... +matches any sequence of characters including slashes. .TP .BI -f " vacfile The results of @@ -127,8 +155,10 @@ .TP .B -q Increase the performance of the +.B -a +or .B -d -option by detecting unchanged files based on a match of the files name and other meta data, +options by detecting unchanged files based on a match of the files name and other meta data, rather than examining the contents of the files. .TP .B -s @@ -137,6 +167,27 @@ .B -v Produce more verbose output on standard error, including the name of the files added to the archive and the vac archives that are expanded and merged. +.TP +.BI -x " excfile +Read exclude patterns from the file +.IR excfile . +Blank lines and lines beginning with +.B # +are ignored. +All other lines should be of the form +.B include +.I pattern +or +.B exclude +.I pattern . +When considering whether to include a directory or file +in the vac archive, +the earliest matching pattern in the file +applies. +The patterns are the same syntax accepted by the +.B -e +option. +This option may be repeated multiple times. .PP .I Unvac lists or extracts files stored in the vac archive @@ -156,9 +207,12 @@ .B -c Write extracted files to standard output instead of creating a file. .TP -.B -h -as per -.IR vac . +.B -d +Reduce the number of blocks read from Venti by +comparing the files to be stored with their counterparts +in the file system. +This option cannot be used with +.BR -c . .TP .B -t Print a list of the files to standard output rather than extracting them. diff -r 88ea8de5bdf7 sys/man/1/venti --- a/sys/man/1/venti Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/1/venti Tue May 29 00:00:00 2012 +0200 @@ -28,7 +28,7 @@ .br .B venti/copy [ -.B -fir +.B -fimrVv ] [ .B -t @@ -37,9 +37,6 @@ .I srchost .I dsthost .I score -[ -.I type -] .SH DESCRIPTION Venti is a SHA1-addressed block storage server. See @@ -102,14 +99,35 @@ to the server .IR dsthost . .PP +Venti's blocks are arranged in a directed acyclic graph (see venti(6)); +there may be multiple paths from a root score to an +interior block (for example, if the same file contents are stored +under multiple names in an archive). +.I Copy +runs more efficiently if it does not copy blocks +(and all their children) multiple times. The .B -f option causes .I copy -to run in `fast' mode, -assuming that if a block already exists on the -destination Venti server, all its children also -exist and need not be checked. +to assume that if a block already exists on the destination +Venti server, all its children also exist and need not be considered. +The +.B -m +option causes +.I copy +to maintain an in-memory list of blocks it has copied +and avoid considering the same block multiple times. +The +.B -f +option is only useful if the destination Venti server is +known not to have lost any blocks due to disk corruption +or other failures. +The +.B -m +option is only useful if enough memory is available to +hold the block list, which typically requires about 1% +of the total number of bytes being copied. .PP The .B -i @@ -138,6 +156,14 @@ replaces pointers to unreadable blocks with pointers to the zero block. It writes the new root score to standard output. +The +.B -v +option prints scores as it copies them, total writes, and other +debugging information. +The +.B -V +option prints debugging information about the Venti protocol +messages send/received. .SH SOURCE .B /sys/src/cmd/venti .SH SEE ALSO diff -r 88ea8de5bdf7 sys/man/2/venti --- a/sys/man/2/venti Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/2/venti Tue May 29 00:00:00 2012 +0200 @@ -2,6 +2,7 @@ .SH NAME venti \- archival storage server .SH SYNOPSIS +.PP .ft L #include .br @@ -10,6 +11,8 @@ #include .SH DESCRIPTION The Venti library provides support for writing Venti servers and clients. +This manual page describes general utility functions. +.PP Other manual pages describe the library functions in detail. .PP .IR Venti-cache (2) diff -r 88ea8de5bdf7 sys/man/2/venti-cache --- a/sys/man/2/venti-cache Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/2/venti-cache Tue May 29 00:00:00 2012 +0200 @@ -7,7 +7,6 @@ vtblockwrite, vtcachealloc, vtcacheallocblock, -vtcacheblocksize, vtcachefree, vtcacheglobal, vtcachelocal, @@ -36,28 +35,25 @@ .ta +\w'\fLVtBlock* 'u +\w'\fLxxxxxxxx'u .PP .B -VtCache* vtcachealloc(VtConn *z, int blocksize, ulong nblocks); +VtCache* vtcachealloc(VtConn *z, ulong maxmem); .PP .B void vtcachefree(VtCache *c); .PP .B -u32int vtcacheblocksize(VtCache *c); -.PP -.B u32int vtglobaltolocal(uchar score[VtScoreSize]) .br .B void vtlocaltoglobal(u32int local, uchar score[VtScoreSize]) .PP .B -VtBlock* vtcacheallocblock(VtCache *c, int type); +VtBlock* vtcacheallocblock(VtCache *c, int type, ulong size); .PP .B VtBlock* vtcachelocal(VtCache *c, u32int addr, int type); .PP .B -VtBlock* vtcacheglobal(VtCache *c, uchar[VtScoreSize], int type); +VtBlock* vtcacheglobal(VtCache *c, uchar[VtScoreSize], int type, ulong size); .PP .B void vtblockput(VtBlock *b); @@ -119,17 +115,13 @@ .IR venti-conn (2) and .IR venti-client (2)), -with room for -.I nblocks -of maximum block size -.I blocksize . +with +.I maxmem +bytes of memory. .PP .I Vtcachefree frees a cache and all the associated blocks. .PP -.I Vtcacheblocksize -returns the cache's maximum block size. -.PP .I Vtglobaltolocal returns the local address corresponding to the given local @@ -147,7 +139,9 @@ .PP .I Vtcacheallocblock allocates a new local block with the given -.IR type . +.I type +and +.IR size . .PP .I Vtcachelocal retrieves the local block at address @@ -160,9 +154,10 @@ .PP .I Vtcacheglobal retrieves the block with the given -.I score +.IR score , +.I dtype and -.I dtype +.I size from the cache, consulting the Venti server if necessary. If passed a local score, diff -r 88ea8de5bdf7 sys/man/2/venti-conn --- a/sys/man/2/venti-conn Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/2/venti-conn Tue May 29 00:00:00 2012 +0200 @@ -28,9 +28,15 @@ VtConn* vtconn(int infd, int outfd) .PP .B +int vtreconn(VtConn *z, int infd, int outfd) +.PP +.B VtConn* vtdial(char *addr) .PP .B +int vtredial(VtConn *z, char *addr) +.PP +.B int vtversion(VtConn *z) .PP .B diff -r 88ea8de5bdf7 sys/man/4/vacfs --- a/sys/man/4/vacfs Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/4/vacfs Tue May 29 00:00:00 2012 +0200 @@ -7,10 +7,6 @@ .B -dips ] [ -.B -c -.I cachesize -] -[ .B -h .I host ] @@ -22,6 +18,10 @@ .B -S .I srvname ] +[ +.B -M +.I mem +] .I vacfile .SH DESCRIPTION .I Vacfs @@ -41,11 +41,6 @@ Options to .I vacfs are: -.TF "-c\fI cachesize" -.PD -.TP -.BI -c " cachesize -The number of file system blocks to cache in memory. The default is 1000 blocks. .TP .B -d Print debugging information to standard error. @@ -83,6 +78,10 @@ rather than mounting it on .IR mtpt . +.TP +.BI -M " mem +The amount of memory, in bytes, allocated to the block cache. The default is 16M. +.PD .SH SOURCE .B /sys/src/cmd/vac .SH "SEE ALSO" diff -r 88ea8de5bdf7 sys/man/6/venti --- a/sys/man/6/venti Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/6/venti Tue May 29 00:00:00 2012 +0200 @@ -434,6 +434,21 @@ upon receiving the .BR VtTgoodbye message, the server terminates up the connection. +.PP +Version +.B 04 +of the Venti protocol is similar to version +.B 02 +(described above) +but has two changes to accomodates larger payloads. +First, it replaces the leading 2-byte packet size with +a 4-byte size. +Second, the +.I count +in the +.B VtTread +packet may be either 2 or 4 bytes; +the total packet length distinguishes the two cases. .SH SEE ALSO .IR venti (1), .IR venti (2), diff -r 88ea8de5bdf7 sys/man/8/venti-fmt --- a/sys/man/8/venti-fmt Sat May 26 00:00:00 2012 +0200 +++ b/sys/man/8/venti-fmt Tue May 29 00:00:00 2012 +0200 @@ -13,7 +13,7 @@ .PP .B venti/fmtarenas [ -.B -Z +.B -4Z ] [ .B -a diff -r 88ea8de5bdf7 sys/src/cmd/vac/file.c --- a/sys/src/cmd/vac/file.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/vac/file.c Tue May 29 00:00:00 2012 +0200 @@ -1735,7 +1735,7 @@ static char EBadVacFormat[] = "bad format for vac file"; static VacFs * -vacfsalloc(VtConn *z, int bsize, int ncache, int mode) +vacfsalloc(VtConn *z, int bsize, ulong cachemem, int mode) { VacFs *fs; @@ -1743,7 +1743,7 @@ fs->z = z; fs->bsize = bsize; fs->mode = mode; - fs->cache = vtcachealloc(z, bsize, ncache); + fs->cache = vtcachealloc(z, cachemem); return fs; } @@ -1772,7 +1772,7 @@ } VacFs* -vacfsopen(VtConn *z, char *file, int mode, int ncache) +vacfsopen(VtConn *z, char *file, int mode, ulong cachemem) { int fd; uchar score[VtScoreSize]; @@ -1793,11 +1793,12 @@ } close(fd); } - return vacfsopenscore(z, score, mode, ncache); +if(debug) fprint(2, "vacfsopen %V\n", score); + return vacfsopenscore(z, score, mode, cachemem); } VacFs* -vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache) +vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem) { VacFs *fs; int n; @@ -1808,29 +1809,42 @@ VtEntry e; n = vtread(z, score, VtRootType, buf, VtRootSize); - if(n < 0) + if(n < 0) { +if(debug) fprint(2, "read %r\n"); return nil; + } if(n != VtRootSize){ werrstr("vtread on root too short"); +if(debug) fprint(2, "size %d\n", n); return nil; } - if(vtrootunpack(&rt, buf) < 0) + if(vtrootunpack(&rt, buf) < 0) { +if(debug) fprint(2, "unpack: %r\n"); return nil; + } if(strcmp(rt.type, "vac") != 0) { +if(debug) fprint(2, "bad type %s\n", rt.type); werrstr("not a vac root"); return nil; } - fs = vacfsalloc(z, rt.blocksize, ncache, mode); + fs = vacfsalloc(z, rt.blocksize, cachemem, mode); memmove(fs->score, score, VtScoreSize); fs->mode = mode; memmove(e.score, rt.score, VtScoreSize); e.gen = 0; - e.psize = rt.blocksize; + + // Don't waste cache memory on pointer blocks + // when rt.blocksize is large. + e.psize = (rt.blocksize/VtEntrySize)*VtEntrySize; + if(e.psize > 60000) + e.psize = (60000/VtEntrySize)*VtEntrySize; + e.dsize = rt.blocksize; +if(debug) fprint(2, "openscore %d psize %d dsize %d\n", (int)rt.blocksize, (int)e.psize, (int)e.dsize); e.type = VtDirType; e.flags = VtEntryActive; e.size = 3*VtEntrySize; @@ -1930,7 +1944,7 @@ * Create a fresh vac fs. */ VacFs * -vacfscreate(VtConn *z, int bsize, int ncache) +vacfscreate(VtConn *z, int bsize, ulong cachemem) { VacFs *fs; VtFile *f; @@ -1941,26 +1955,26 @@ VacDir vd; MetaEntry me; int psize; - int mbsize; - if((fs = vacfsalloc(z, bsize, ncache, VtORDWR)) == nil) + if((fs = vacfsalloc(z, bsize, cachemem, VtORDWR)) == nil) return nil; - + /* * Fake up an empty vac fs. */ - psize = bsize; + psize = bsize/VtScoreSize*VtScoreSize; + if(psize > 60000) + psize = 60000/VtScoreSize*VtScoreSize; +if(debug) fprint(2, "create bsize %d psize %d\n", bsize, psize); + f = vtfilecreateroot(fs->cache, psize, bsize, VtDirType); + if(f == nil) + sysfatal("vtfilecreateroot: %r"); vtfilelock(f, VtORDWR); - - /* Metablocks can't be too big -- they have 16-bit offsets in them. */ - mbsize = bsize; - if(mbsize >= 56*1024) - mbsize = 56*1024; /* Write metablock containing root directory VacDir. */ - b = vtcacheallocblock(fs->cache, VtDataType); - mbinit(&mb, b->data, mbsize, mbsize/BytesPerEntry); + b = vtcacheallocblock(fs->cache, VtDataType, bsize); + mbinit(&mb, b->data, bsize, bsize/BytesPerEntry); memset(&vd, 0, sizeof vd); vd.elem = "/"; vd.mode = 0777|ModeDir; @@ -1988,13 +2002,12 @@ /* Second entry: empty metadata stream. */ e.type = VtDataType; - e.dsize = mbsize; vtentrypack(&e, buf, 0); vtfilewrite(f, buf, VtEntrySize, VtEntrySize); /* Third entry: metadata stream with root directory. */ memmove(e.score, metascore, VtScoreSize); - e.size = mbsize; + e.size = bsize; vtentrypack(&e, buf, 0); vtfilewrite(f, buf, VtEntrySize, VtEntrySize*2); diff -r 88ea8de5bdf7 sys/src/cmd/vac/pack.c --- a/sys/src/cmd/vac/pack.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/vac/pack.c Tue May 29 00:00:00 2012 +0200 @@ -77,7 +77,7 @@ magic = U32GET(p); if(magic != MetaMagic && magic != MetaMagic+1) { - werrstr("bad meta block magic"); + werrstr("bad meta block magic %#08ux", magic); return -1; } mb->size = U16GET(p+4); diff -r 88ea8de5bdf7 sys/src/cmd/vac/unvac.c --- a/sys/src/cmd/vac/unvac.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/vac/unvac.c Tue May 29 00:00:00 2012 +0200 @@ -92,7 +92,7 @@ if(vtconnect(conn) < 0) sysfatal("vtconnect: %r"); - fs = vacfsopen(conn, argv[0], VtOREAD, 128); + fs = vacfsopen(conn, argv[0], VtOREAD, 4<<20); if(fs == nil) sysfatal("vacfsopen: %r"); @@ -225,7 +225,7 @@ if(!table && !tostdout && vdir){ // create directory if((dp = dirstat(name)) == nil){ - if((fd = create(name, OREAD, DMDIR|(mode&0777))) < 0){ + if((fd = create(name, OREAD, DMDIR|0700|(mode&0777))) < 0){ fprint(2, "mkdir %s: %r\n", name); vdeclose(vde); } diff -r 88ea8de5bdf7 sys/src/cmd/vac/vac.c --- a/sys/src/cmd/vac/vac.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/vac/vac.c Tue May 29 00:00:00 2012 +0200 @@ -15,6 +15,7 @@ enum { BlockSize = 8*1024, + CacheSize = 4<<20, }; struct @@ -81,8 +82,6 @@ u = unittoull(EARGF(usage())); if(u < 512) u = 512; - if(u > VtMaxLumpSize) - u = VtMaxLumpSize; blocksize = u; break; case 'd': @@ -151,10 +150,10 @@ if((outfd = create(archivefile, OWRITE, 0666)) < 0) sysfatal("create %s: %r", archivefile); atexit(removevacfile); // because it is new - if((fs = vacfscreate(z, blocksize, 512)) == nil) + if((fs = vacfscreate(z, blocksize, CacheSize)) == nil) sysfatal("vacfscreate: %r"); }else{ - if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil) + if((fs = vacfsopen(z, archivefile, VtORDWR, CacheSize)) == nil) sysfatal("vacfsopen %s: %r", archivefile); if((fdiff = recentarchive(fs, oldpath)) != nil){ if(verbose) @@ -194,13 +193,13 @@ else if((outfd = create(vacfile, OWRITE, 0666)) < 0) sysfatal("create %s: %r", vacfile); atexit(removevacfile); - if((fs = vacfscreate(z, blocksize, 512)) == nil) + if((fs = vacfscreate(z, blocksize, CacheSize)) == nil) sysfatal("vacfscreate: %r"); f = vacfsgetroot(fs); fdiff = nil; if(diffvac){ - if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil) + if((fsdiff = vacfsopen(z, diffvac, VtOREAD, CacheSize)) == nil) warn("vacfsopen %s: %r", diffvac); else fdiff = vacfsgetroot(fsdiff); @@ -414,7 +413,7 @@ vac(VacFile *fp, VacFile *diffp, char *name, Dir *d) { char *elem, *s; - static char buf[65536]; + static char *buf; int fd, i, n, bsize; vlong off; Dir *dk; // kids @@ -461,7 +460,11 @@ if(vacfilesetdir(f, &vd) < 0) warn("vacfilesetdir %s: %r", name); - + + bsize = fs->bsize; + if(buf == nil) + buf = vtmallocz(bsize); + if(d->mode&DMDIR){ while((n = dirread(fd, &dk)) > 0){ for(i=0; ibsize; if(fdiff){ /* * Copy fdiff's contents into f by moving the score. @@ -644,7 +646,7 @@ if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0) return -1; - if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil) + if((mfs = vacfsopen(z, name, VtOREAD, CacheSize)) == nil) return -1; if(verbose) fprint(2, "merging %s\n", name); diff -r 88ea8de5bdf7 sys/src/cmd/vac/vac.h --- a/sys/src/cmd/vac/vac.h Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/vac/vac.h Tue May 29 00:00:00 2012 +0200 @@ -95,9 +95,9 @@ VtCache *cache; }; -VacFs *vacfsopen(VtConn *z, char *file, int mode, int ncache); -VacFs *vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache); -VacFs *vacfscreate(VtConn *z, int bsize, int ncache); +VacFs *vacfsopen(VtConn *z, char *file, int mode, ulong cachemem); +VacFs *vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem); +VacFs *vacfscreate(VtConn *z, int bsize, ulong cachemem); void vacfsclose(VacFs *fs); int vacfssync(VacFs *fs); int vacfssnapshot(VacFs *fs, char *src, char *dst); diff -r 88ea8de5bdf7 sys/src/cmd/vac/vacfs.c --- a/sys/src/cmd/vac/vacfs.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/vac/vacfs.c Tue May 29 00:00:00 2012 +0200 @@ -2,16 +2,10 @@ #include #include "vac.h" -#define convM2Su(a, b, c, d) convM2S(a, b, c) -#define convS2Mu(a, b, c, d) convS2M(a, b, c) -#define convM2Du(a, b, c, d) convM2D(a, b, c) -#define convD2Mu(a, b, c, d) convD2M(a, b, c) - typedef struct Fid Fid; enum { - Stacksize = 320 * 1024, /* was 32K */ OPERM = 0x3 /* mask of all permission types in open mode */ }; @@ -49,7 +43,6 @@ VacFs *fs; VtConn *conn; int noperm; -int dotu; char *defmnt; Fid * newfid(int); @@ -117,6 +110,31 @@ noted(NDFLT); } +#define TWID64 ~(u64int)0 +static u64int +unittoull(char *s) +{ + char *es; + u64int n; + + if(s == nil) + return TWID64; + n = strtoul(s, &es, 0); + if(*es == 'k' || *es == 'K'){ + n *= 1024; + es++; + }else if(*es == 'm' || *es == 'M'){ + n *= 1024*1024; + es++; + }else if(*es == 'g' || *es == 'G'){ + n *= 1024*1024*1024; + es++; + } + if(*es != '\0') + return TWID64; + return n; +} + void threadmain(int argc, char *argv[]) { @@ -124,10 +142,10 @@ int p[2], fd; int stdio; char *host = nil; - long ncache; + ulong mem; + mem = 16<<20; stdio = 0; - ncache = 256; fmtinstall('H', encodefmt); fmtinstall('V', vtscorefmt); fmtinstall('F', vtfcallfmt); @@ -139,9 +157,6 @@ fmtinstall('F', fcallfmt); dflag = 1; break; - case 'c': - ncache = atoi(EARGF(usage())); - break; case 'i': defmnt = nil; stdio = 1; @@ -157,6 +172,9 @@ case 's': defsrv = "vacfs"; break; + case 'M': + mem = unittoull(EARGF(usage())); + break; case 'm': defmnt = EARGF(usage()); break; @@ -190,7 +208,7 @@ if(vtconnect(conn) < 0) sysfatal("vtconnect: %r"); - fs = vacfsopen(conn, argv[0], VtOREAD, ncache); + fs = vacfsopen(conn, argv[0], VtOREAD, mem); if(fs == nil) sysfatal("vacfsopen: %r"); @@ -202,7 +220,7 @@ srvfd = p[1]; } - procrfork(srv, 0, Stacksize, RFFDG|RFNAMEG|RFNOTEG); + procrfork(srv, 0, 32 * 1024, RFFDG|RFNAMEG|RFNOTEG); if(!stdio){ close(p[0]); @@ -234,7 +252,7 @@ void usage(void) { - fprint(2, "usage: %s [-sd] [-h host] [-c ncache] [-m mountpoint] vacfile\n", argv0); + fprint(2, "usage: %s [-sd] [-h host] [-m mountpoint] [-M mem] vacfile\n", argv0); threadexitsall("usage"); } @@ -258,10 +276,6 @@ if(strncmp(rhdr.version, "9P2000", 6) != 0) return vtstrdup("unrecognized 9P version"); thdr.version = "9P2000"; - if(strncmp(rhdr.version, "9P2000.u", 8) == 0){ - dotu = 1; - thdr.version = "9P2000.u"; - } return nil; } @@ -634,7 +648,7 @@ dir.gid = vd->gid; dir.muid = vd->mid; - ret = convD2Mu(&dir, p, np, dotu); + ret = convD2M(&dir, p, np); return ret; } @@ -708,7 +722,7 @@ n = read9pmsg(mfd[0], mdata, sizeof mdata); if(n <= 0) break; - if(convM2Su(mdata, n, &rhdr, dotu) != n) + if(convM2S(mdata, n, &rhdr) != n) sysfatal("convM2S conversion error"); if(dflag) @@ -729,9 +743,9 @@ thdr.tag = rhdr.tag; if(dflag) fprint(2, "vacfs:->%F\n", &thdr); - n = convS2Mu(&thdr, mdata, messagesize, dotu); + n = convS2M(&thdr, mdata, messagesize); if(n <= BIT16SZ) - sysfatal("convS2Mu conversion error"); + sysfatal("convS2M conversion error"); if(err) vtfree(err); diff -r 88ea8de5bdf7 sys/src/cmd/venti/copy.c --- a/sys/src/cmd/venti/copy.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/copy.c Tue May 29 00:00:00 2012 +0200 @@ -2,9 +2,16 @@ #include #include #include +#include #include #include +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65535, +}; + int changes; int rewrite; int ignoreerrors; @@ -71,12 +78,12 @@ void usage(void) { - fprint(2, "usage: %s [-fimrv] [-t type] srchost dsthost score\n", argv0); - exits("usage"); + fprint(2, "usage: %s [-fimrVv] [-t type] srchost dsthost score\n", argv0); + threadexitsall("usage"); } void -walk(uchar score[VtScoreSize], uint type, int base) +walk(uchar score[VtScoreSize], uint type, int base, int depth) { int i, n; uchar *buf; @@ -84,6 +91,12 @@ VtEntry e; VtRoot root; + if(verbose){ + for(i = 0; i < depth; i++) + fprint(2, " "); + fprint(2, "-> %d %d %d %V\n", depth, type, base, score); + } + if(memcmp(score, vtzeroscore, VtScoreSize) == 0 || memcmp(score, zeroscore, VtScoreSize) == 0) return; @@ -116,8 +129,8 @@ fprint(2, "warning: could not unpack root in %V %d\n", score, type); break; } - walk(root.prev, VtRootType, 0); - walk(root.score, VtDirType, 0); + walk(root.prev, VtRootType, 0, depth+1); + walk(root.score, VtDirType, 0, depth+1); if(rewrite) vtrootpack(&root, buf); /* walk might have changed score */ break; @@ -130,7 +143,7 @@ } if(!(e.flags & VtEntryActive)) continue; - walk(e.score, e.type, e.type&VtTypeBaseMask); + walk(e.score, e.type, e.type&VtTypeBaseMask, depth+1); /* * Don't repack unless we're rewriting -- some old * vac files have psize==0 and dsize==0, and these @@ -149,7 +162,7 @@ default: /* pointers */ for(i=0; i +#include +#include +#include +#include +#include + +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65535, +}; + +VtConn *z; +char *host; + +void +usage(void) +{ + fprint(2, "usage: venti/dump [-h host] score\n"); + threadexitsall("usage"); +} + +Biobuf bout; +char spaces[256]; + +void +dump(int indent, uchar *score, int type) +{ + int i, n; + uchar *buf; + VtEntry e; + VtRoot root; + + if(spaces[0] == 0) + memset(spaces, ' ', sizeof spaces-1); + + buf = vtmallocz(VtMaxLumpSize); + if(memcmp(score, vtzeroscore, VtScoreSize) == 0) + n = 0; + else + n = vtread(z, score, type, buf, VtMaxLumpSize); + if(n < 0){ + Bprint(&bout, "%.*serror reading %V: %r\n", indent*4, spaces, score); + goto out; + } + switch(type){ + case VtRootType: + if(vtrootunpack(&root, buf) < 0){ + Bprint(&bout, "%.*serror unpacking root %V: %r\n", indent*4, spaces, score); + goto out; + } + Bprint(&bout, "%.*s%V root name=%s type=%s prev=%V bsize=%ld\n", + indent*4, spaces, score, root.name, root.type, root.prev, root.blocksize); + dump(indent+1, root.score, VtDirType); + break; + + case VtDirType: + Bprint(&bout, "%.*s%V dir n=%d\n", indent*4, spaces, score, n); + for(i=0; i*VtEntrySize= VtDirType) + Bprint(&bout, "%.*s%V dir+%d\n", indent*4, spaces, score, type-VtDirType); + else + Bprint(&bout, "%.*s%V data+%d\n", indent*4, spaces, score, type-VtDirType); + for(i=0; i= 0) + goto havetype; + } + sysfatal("cannot find block %V", score); + +havetype: + Binit(&bout, 1, OWRITE); + dump(0, score, type); + Bflush(&bout); + threadexitsall(nil); +} diff -r 88ea8de5bdf7 sys/src/cmd/venti/mkfile --- a/sys/src/cmd/venti/mkfile Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/mkfile Tue May 29 00:00:00 2012 +0200 @@ -6,6 +6,7 @@ ro\ sync\ write\ + dump\ BIN=/$objtype/bin/venti @@ -14,7 +15,7 @@ CFLAGS=$CFLAGS -I. -extra:V: $O.devnull $O.mkroot $O.randtest $O.readlist $O.root +extra:V: $O.devnull $O.mkroot $O.randtest $O.readlist $O.ro $O.root all:V: srv.all.dir install:V: srv.install.dir diff -r 88ea8de5bdf7 sys/src/cmd/venti/randtest.c --- a/sys/src/cmd/venti/randtest.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/randtest.c Tue May 29 00:00:00 2012 +0200 @@ -96,8 +96,9 @@ buf = vtmalloc(blocksize); cur = 0; packets = totalbytes/blocksize; - if(maxpackets == 0) - maxpackets = packets; + if(maxpackets > 0 && maxpackets < packets) + packets = maxpackets; + totalbytes = (vlong)packets * blocksize; order = vtmalloc(packets*sizeof order[0]); for(i=0; i #include +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65535, +}; + void usage(void) { diff -r 88ea8de5bdf7 sys/src/cmd/venti/readfile.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/src/cmd/venti/readfile.c Tue May 29 00:00:00 2012 +0200 @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include + +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65535, +}; + +int chatty; + +void +usage(void) +{ + fprint(2, "usage: readfile [-v] [-h host] score\n"); + threadexitsall("usage"); +} + +void +threadmain(int argc, char *argv[]) +{ + int n; + uchar score[VtScoreSize]; + uchar *buf; + char *host, *type; + vlong off; + VtEntry e; + VtRoot root; + VtCache *c; + VtConn *z; + VtFile *f; + + quotefmtinstall(); + fmtinstall('F', vtfcallfmt); + fmtinstall('V', vtscorefmt); + + host = nil; + ARGBEGIN{ + case 'V': + chattyventi++; + break; + case 'h': + host = EARGF(usage()); + break; + case 'v': + chatty++; + break; + default: + usage(); + break; + }ARGEND + + if(argc != 1) + usage(); + + type = nil; + if(vtparsescore(argv[0], &type, score) < 0) + sysfatal("could not parse score '%s': %r", argv[0]); + if(type == nil || strcmp(type, "file") != 0) + sysfatal("bad score - not file:..."); + + buf = vtmallocz(VtMaxLumpSize); + + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + // root block ... + n = vtread(z, score, VtRootType, buf, VtMaxLumpSize); + if(n < 0) + sysfatal("could not read root %V: %r", score); + if(n != VtRootSize) + sysfatal("root block %V is wrong size %d != %d", score, n, VtRootSize); + if(vtrootunpack(&root, buf) < 0) + sysfatal("unpacking root block %V: %r", score); + if(strcmp(root.type, "file") != 0) + sysfatal("bad root type %q (not 'file')", root.type); + if(chatty) + fprint(2, "%V: %q %q %V %d %V\n", + score, root.name, root.type, + root.score, root.blocksize, root.prev); + + // ... points at entry block + n = vtread(z, root.score, VtDirType, buf, VtMaxLumpSize); + if(n < 0) + sysfatal("could not read entry %V: %r", root.score); + if(n != VtEntrySize) + sysfatal("dir block %V is wrong size %d != %d", root.score, n, VtEntrySize); + if(vtentryunpack(&e, buf, 0) < 0) + sysfatal("unpacking dir block %V: %r", root.score); + if((e.type&VtTypeBaseMask) != VtDataType) + sysfatal("not a single file"); + + // open and read file + c = vtcachealloc(z, root.blocksize*32); + if(c == nil) + sysfatal("vtcachealloc: %r"); + f = vtfileopenroot(c, &e); + if(f == nil) + sysfatal("vtfileopenroot: %r"); + off = 0; + vtfilelock(f, VtOREAD); + while((n = vtfileread(f, buf, VtMaxLumpSize, off)) > 0){ + write(1, buf, n); + off += n; + } + threadexitsall(0); +} diff -r 88ea8de5bdf7 sys/src/cmd/venti/readlist.c --- a/sys/src/cmd/venti/readlist.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/readlist.c Tue May 29 00:00:00 2012 +0200 @@ -4,6 +4,12 @@ #include #include +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65535, +}; + char *host; Biobuf b; VtConn *z; diff -r 88ea8de5bdf7 sys/src/cmd/venti/root.c --- a/sys/src/cmd/venti/root.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/root.c Tue May 29 00:00:00 2012 +0200 @@ -4,6 +4,12 @@ #include #include +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65535, +}; + void usage(void) { diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/arena.c --- a/sys/src/cmd/venti/srv/arena.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/arena.c Tue May 29 00:00:00 2012 +0200 @@ -31,7 +31,9 @@ { needzeroscore(); /* OS X */ + qlock(&sumlock); sumwait.l = &sumlock; + qunlock(&sumlock); if(vtproc(sumproc, nil) < 0){ seterr(EOk, "can't start arena checksum slave: %r"); @@ -478,9 +480,6 @@ { ASum *as; - if(sumwait.l == nil) - return; - as = MK(ASum); if(as == nil) return; @@ -492,7 +491,12 @@ else sumq = as; sumqtail = as; - rwakeup(&sumwait); + /* + * Might get here while initializing arenas, + * before initarenasum has been called. + */ + if(sumwait.l) + rwakeup(&sumwait); qunlock(&sumlock); } @@ -513,7 +517,6 @@ qunlock(&sumlock); arena = as->arena; free(as); - sumarena(arena); } } @@ -683,9 +686,8 @@ logerr(ECorrupt, "arena tail name %s head %s", arena->name, head.name); else if(arena->clumpmagic != head.clumpmagic) - logerr(ECorrupt, "arena %d tail clumpmagic 0x%lux head 0x%lux", - debugarena, (ulong)arena->clumpmagic, - (ulong)head.clumpmagic); + logerr(ECorrupt, "arena tail clumpmagic 0x%lux head 0x%lux", + (ulong)arena->clumpmagic, (ulong)head.clumpmagic); else if(arena->version != head.version) logerr(ECorrupt, "arena tail version %d head version %d", arena->version, head.version); diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/bloom.c --- a/sys/src/cmd/venti/srv/bloom.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/bloom.c Tue May 29 00:00:00 2012 +0200 @@ -229,6 +229,22 @@ runlock(&b->lk); } +void +markbloomfiltern(Bloom *b, u8int score[][20], int n) +{ + int i; + + if(b == nil || b->data == nil) + return; + + rlock(&b->lk); + qlock(&b->mod); + for(i=0; imod); + runlock(&b->lk); +} + static void bloomwriteproc(void *v) { diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/buildindex.c --- a/sys/src/cmd/venti/srv/buildindex.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/buildindex.c Tue May 29 00:00:00 2012 +0200 @@ -11,6 +11,20 @@ MaxBufSize = 4*1024*1024, }; +typedef struct IEntryBuf IEntryBuf; +struct IEntryBuf +{ + IEntry ie[100]; + int nie; +}; + +typedef struct ScoreBuf ScoreBuf; +struct ScoreBuf +{ + uchar score[100][VtScoreSize]; + int nscore; +}; + int dumb; int errors; char **isect; @@ -36,7 +50,7 @@ void usage(void) { - fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n"); + fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n"); threadexitsall("usage"); } @@ -121,10 +135,10 @@ /* start index procs */ fprint(2, "%T read index\n"); - isectdonechan = chancreate(sizeof(void*), 0); + isectdonechan = chancreate(sizeof(void*), 1); for(i=0; insects; i++){ if(shouldprocess(ix->sects[i])){ - ix->sects[i]->writechan = chancreate(sizeof(IEntry), 0); + ix->sects[i]->writechan = chancreate(sizeof(IEntryBuf), 1); vtproc(isectproc, ix->sects[i]); } } @@ -216,12 +230,17 @@ ClumpInfo *ci, *cis; IEntry ie; Part *p; + IEntryBuf *buf, *b; + uchar *score; + ScoreBuf sb; p = v; threadsetname("arenaproc %s", p->name); + buf = MKNZ(IEntryBuf, ix->nsects); nskip = 0; tot = 0; + sb.nscore = 0; cis = MKN(ClumpInfo, ClumpChunks); for(i=0; inarenas; i++){ a = ix->arenas[i]; @@ -260,10 +279,23 @@ tot++; x = indexsect(ix, ie.score); assert(0 <= x && x < ix->nsects); - if(ix->sects[x]->writechan) - send(ix->sects[x]->writechan, &ie); - if(ix->bloom) - markbloomfilter(ix->bloom, ie.score); + if(ix->sects[x]->writechan) { + b = &buf[x]; + b->ie[b->nie] = ie; + b->nie++; + if(b->nie == nelem(b->ie)) { + send(ix->sects[x]->writechan, b); + b->nie = 0; + } + } + if(ix->bloom) { + score = sb.score[sb.nscore++]; + scorecp(score, ie.score); + if(sb.nscore == nelem(sb.score)) { + markbloomfiltern(ix->bloom, sb.score, sb.nscore); + sb.nscore = 0; + } + } } } } @@ -272,6 +304,14 @@ } add(&arenaentries, tot); add(&skipentries, nskip); + + for(i=0; insects; i++) + if(ix->sects[i]->writechan && buf[i].nie > 0) + send(ix->sects[i]->writechan, &buf[i]); + free(buf); + free(cis); + if(ix->bloom && sb.nscore > 0) + markbloomfiltern(ix->bloom, sb.score, sb.nscore); sendp(arenadonechan, p); } @@ -751,6 +791,7 @@ uchar *data, *p; Buf *buf; IEntry ie; + IEntryBuf ieb; IPool *ipool; ISect *is; Minibuf *mbuf, *mb; @@ -813,7 +854,7 @@ } if (nbuf == 0) { fprint(2, "%s: brand-new index, no work to do\n", argv0); - threadexitsall(0); + threadexitsall(nil); } /* size buffer to use extra memory */ @@ -850,14 +891,17 @@ assert(p == data+nbuf*bufsize); n = 0; - while(recv(is->writechan, &ie) == 1){ - if(ie.ia.addr == 0) + while(recv(is->writechan, &ieb) == 1){ + if(ieb.nie == 0) break; - buck = score2bucket(is, ie.score); - i = buck/bufbuckets; - assert(i < nbuf); - bwrite(&buf[i], &ie); - n++; + for(j=0; j 0 && strcmp(flds[0], "mgr") == 0){ + /* do nothing */ + }else if(i == 2 && strcmp(flds[0], "isect") == 0){ sv = MKN(ISect*, config->nsects + 1); for(i = 0; i < config->nsects; i++) sv[i] = config->sects[i]; diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/dat.h --- a/sys/src/cmd/venti/srv/dat.h Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/dat.h Tue May 29 00:00:00 2012 +0200 @@ -37,6 +37,12 @@ enum { + /* + * formerly fundamental constant, + * now a server-imposed limitation. + */ + VtMaxLumpSize = 56*1024, + ABlockLog = 9, /* log2(512), the quantum for reading arenas */ ANameSize = 64, MaxDiskBlock = 64*1024, /* max. allowed size for a disk block */ @@ -734,6 +740,7 @@ extern u8int zeroscore[VtScoreSize]; extern int compressblocks; extern int writestodevnull; /* dangerous - for performance debugging */ +extern int bootstrap; /* writes but does not index - cannot read */ extern int collectstats; extern QLock memdrawlock; extern int icachesleeptime; diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtarenas.c --- a/sys/src/cmd/venti/srv/fmtarenas.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/fmtarenas.c Tue May 29 00:00:00 2012 +0200 @@ -5,7 +5,7 @@ void usage(void) { - fprint(2, "usage: fmtarenas [-Z] [-b blocksize] [-a arenasize] name file\n"); + fprint(2, "usage: fmtarenas [-4Z] [-a arenasize] [-b blocksize] name file\n"); threadexitsall(0); } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtbloom.c --- a/sys/src/cmd/venti/srv/fmtbloom.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/fmtbloom.c Tue May 29 00:00:00 2012 +0200 @@ -7,7 +7,7 @@ void usage(void) { - fprint(2, "usage: fmtbloom [-s size] [-n nblocks | -N nhash] file\n"); + fprint(2, "usage: fmtbloom [-n nblocks | -N nhash] [-s size] file\n"); threadexitsall(0); } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtindex.c --- a/sys/src/cmd/venti/srv/fmtindex.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/fmtindex.c Tue May 29 00:00:00 2012 +0200 @@ -5,7 +5,7 @@ void usage(void) { - fprint(2, "usage: fmtindex [-a] config\n"); + fprint(2, "usage: fmtindex [-a] venti.conf\n"); threadexitsall(0); } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fmtisect.c --- a/sys/src/cmd/venti/srv/fmtisect.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/fmtisect.c Tue May 29 00:00:00 2012 +0200 @@ -5,7 +5,7 @@ void usage(void) { - fprint(2, "usage: fmtisect [-Z] [-b blocksize] name file\n"); + fprint(2, "usage: fmtisect [-1Z] [-b blocksize] name file\n"); threadexitsall(0); } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/fns.h --- a/sys/src/cmd/venti/srv/fns.h Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/fns.h Tue May 29 00:00:00 2012 +0200 @@ -105,6 +105,7 @@ int lookupscore(u8int *score, int type, IAddr *ia); int maparenas(AMap *am, Arena **arenas, int n, char *what); void markbloomfilter(Bloom*, u8int*); +void markbloomfiltern(Bloom*, u8int[][20], int); uint msec(void); int namecmp(char *s, char *t); void namecp(char *dst, char *src); diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/httpd.c --- a/sys/src/cmd/venti/srv/httpd.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/httpd.c Tue May 29 00:00:00 2012 +0200 @@ -396,6 +396,7 @@ "ignorebloom", &ignorebloom, "syncwrites", &syncwrites, "icacheprefetch", &icacheprefetch, + "bootstrap", &bootstrap, 0 }; @@ -562,6 +563,8 @@ hprint(hout, " mem=sealed"); if(arena->diskstats.sealed) hprint(hout, " disk=sealed"); + if(arena->inqueue) + hprint(hout, " inqueue"); hprint(hout, "\n"); if(scorecmp(zeroscore, arena->score) != 0) hprint(hout, "\tscore=%V\n", arena->score); diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/icache.c --- a/sys/src/cmd/venti/srv/icache.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/icache.c Tue May 29 00:00:00 2012 +0200 @@ -375,6 +375,9 @@ { IEntry *ie; + if(bootstrap) + return -1; + qlock(&icache.lock); addstat(StatIcacheLookup, 1); if((ie = ihashlookup(icache.hash, score, type)) != nil){ @@ -405,6 +408,9 @@ { ISum *toload; + if(bootstrap) + return -1; + qlock(&icache.lock); icacheinsert(score, ia, state); if(state == IEClean) diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/index.c --- a/sys/src/cmd/venti/srv/index.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/index.c Tue May 29 00:00:00 2012 +0200 @@ -75,18 +75,44 @@ blocksize = ix->blocksize; for(i = 0; i < ix->nsects; i++){ is = sects[i]; - if(namecmp(ix->name, is->index) != 0 - || is->blocksize != blocksize - || is->tabsize != tabsize - || namecmp(is->name, ix->smap[i].name) != 0 - || is->start != ix->smap[i].start - || is->stop != ix->smap[i].stop - || last != is->start - || is->start > is->stop){ - seterr(ECorrupt, "inconsistent index sections in %s", ix->name); + if(namecmp(is->index, ix->name) != 0) { + seterr(ECorrupt, "%s: index name is %s, not %s", + sects[i]->part->name, is->index, ix->name); + bad: freeindex(ix); return nil; } + if(is->blocksize != blocksize) { + seterr(ECorrupt, "%s: blocksize is %d, not %d", + sects[i]->part->name, (int)is->blocksize, (int)blocksize); + goto bad; + } + if(is->tabsize != tabsize) { + seterr(ECorrupt, "%s: tabsize is %d, not %d", + sects[i]->part->name, (int)is->tabsize, (int)tabsize); + goto bad; + } + if(namecmp(is->name, ix->smap[i].name) != 0) { + seterr(ECorrupt, "%s: name is %s, not %s", + sects[i]->part->name, is->name, ix->smap[i].name); + goto bad; + } + if(is->start != ix->smap[i].start || is->stop != ix->smap[i].stop) { + seterr(ECorrupt, "%s: range is %lld,%lld, not %lld,%lld", + sects[i]->part->name, is->start, is->stop, + ix->smap[i].start, ix->smap[i].stop); + goto bad; + } + if(is->start > is->stop) { + seterr(ECorrupt, "%s: invalid range %lld,%lld", + sects[i]->part->name, is->start, is->stop); + goto bad; + } + if(is->start != last || is->start > is->stop) { + seterr(ECorrupt, "%s: range %lld-%lld, but last section ended at %lld", + sects[i]->part->name, is->start, is->stop, last); + goto bad; + } last = is->stop; } ix->tabsize = tabsize; @@ -272,11 +298,15 @@ return nil; } if(blocksize != sects[i]->blocksize){ - seterr(EOk, "mismatched block sizes in index sections"); + seterr(EOk, "%s has block size %d, but %s has %d", + sects[0]->part->name, (int)blocksize, + sects[i]->part->name, (int)sects[i]->blocksize); return nil; } if(tabsize != sects[i]->tabsize){ - seterr(EOk, "mismatched config table sizes in index sections"); + seterr(EOk, "%s has table size %d, but %s has %d", + sects[0]->part->name, (int)tabsize, + sects[i]->part->name, (int)sects[i]->tabsize); return nil; } nb += sects[i]->blocks; @@ -288,7 +318,10 @@ for(i = 0; i < n; i++){ for(j = i + 1; j < n; j++){ if(namecmp(sects[i]->name, sects[j]->name) == 0){ - seterr(EOk, "duplicate section name %s for index %s", sects[i]->name, name); + seterr(EOk, "%s and %s both have section name %s", + sects[i]->part->name, + sects[j]->part->name, + sects[i]->name); return nil; } } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/lump.c --- a/sys/src/cmd/venti/srv/lump.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/lump.c Tue May 29 00:00:00 2012 +0200 @@ -2,6 +2,7 @@ #include "dat.h" #include "fns.h" +int bootstrap = 0; int syncwrites = 0; int queuewrites = 0; int writestodevnull = 0; diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/mgr.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/src/cmd/venti/srv/mgr.c Tue May 29 00:00:00 2012 +0200 @@ -0,0 +1,1022 @@ +/* + * mirror manager. + * a work in progress. + * use at your own risk. + */ + +#include "stdinc.h" +#include +#include +#include "dat.h" +#include "fns.h" + +#ifdef PLAN9PORT +#define sp s.sp +#define ep e.ep +#endif + +void sendmail(char *content, char *subject, char *msg); +#define TIME "[0-9]+/[0-9]+ [0-9]+:[0-9]+:[0-9]+" + +char *mirrorregexp = + "^" TIME " (" + "([^ ]+ \\([0-9,]+-[0-9,]+\\))" + "|( copy [0-9,]+-[0-9,]+ (data|hole|directory|tail))" + "|( sha1 [0-9,]+-[0-9,]+)" + "|([^ ]+: [0-9,]+ used mirrored)" + "|([^ \\-]+-[^ \\-]+( mirrored| sealed| empty)+)" + ")$"; +Reprog *mirrorprog; + +char *verifyregexp = + "^" TIME " (" + "([^ ]+: unsealed [0-9,]+ bytes)" + ")$"; +Reprog *verifyprog; + +#undef pipe +enum +{ + LogSize = 4*1024*1024 // TODO: make smaller +}; + +VtLog *errlog; + +typedef struct Mirror Mirror; +struct Mirror +{ + char *src; + char *dst; +}; + +typedef struct Conf Conf; +struct Conf +{ + Mirror *mirror; + int nmirror; + char **verify; + int nverify; + char *httpaddr; + char *webroot; + char *smtp; + char *mailfrom; + char *mailto; + int mirrorfreq; + int verifyfreq; +}; + +typedef struct Job Job; +struct Job +{ + char *name; + QLock lk; + char *argv[10]; + int oldok; + int newok; + VtLog *oldlog; + VtLog *newlog; + int pid; + int pipe; + int nrun; + vlong freq; + vlong runstart; + vlong runend; + double offset; + int (*ok)(char*); +}; + +Job *job; +int njob; +char *bin; + +vlong time0; +Conf conf; + +void +usage(void) +{ + fprint(2, "usage: mgr [-s] [-b bin/venti/] venti.conf\n"); + threadexitsall(0); +} + +int +rdconf(char *file, Conf *conf) +{ + char *s, *line, *flds[10]; + int i, ok; + IFile f; + + if(readifile(&f, file) < 0) + return -1; + memset(conf, 0, sizeof *conf); + ok = -1; + line = nil; + for(;;){ + s = ifileline(&f); + if(s == nil){ + ok = 0; + break; + } + line = estrdup(s); + i = getfields(s, flds, nelem(flds), 1, " \t\r"); + if(i <= 0 || strcmp(flds[0], "mgr") != 0) { + /* do nothing */ + }else if(i == 4 && strcmp(flds[1], "mirror") == 0) { + if(conf->nmirror%64 == 0) + conf->mirror = vtrealloc(conf->mirror, (conf->nmirror+64)*sizeof(conf->mirror[0])); + conf->mirror[conf->nmirror].src = vtstrdup(flds[2]); + conf->mirror[conf->nmirror].dst = vtstrdup(flds[3]); + conf->nmirror++; + }else if(i == 3 && strcmp(flds[1], "mirrorfreq") == 0) { + conf->mirrorfreq = atoi(flds[2]); + }else if(i == 3 && strcmp(flds[1], "verify") == 0) { + if(conf->nverify%64 == 0) + conf->verify = vtrealloc(conf->verify, (conf->nverify+64)*sizeof(conf->verify[0])); + conf->verify[conf->nverify++] = vtstrdup(flds[2]); + }else if(i == 3 && strcmp(flds[1], "verifyfreq") == 0) { + conf->verifyfreq = atoi(flds[2]); + }else if(i == 3 && strcmp(flds[1], "httpaddr") == 0){ + if(conf->httpaddr){ + seterr(EAdmin, "duplicate httpaddr lines in configuration file %s", file); + break; + } + conf->httpaddr = estrdup(flds[2]); + }else if(i == 3 && strcmp(flds[1], "webroot") == 0){ + if(conf->webroot){ + seterr(EAdmin, "duplicate webroot lines in configuration file %s", file); + break; + } + conf->webroot = estrdup(flds[2]); + }else if(i == 3 && strcmp(flds[1], "smtp") == 0) { + if(conf->smtp){ + seterr(EAdmin, "duplicate smtp lines in configuration file %s", file); + break; + } + conf->smtp = estrdup(flds[2]); + }else if(i == 3 && strcmp(flds[1], "mailfrom") == 0) { + if(conf->mailfrom){ + seterr(EAdmin, "duplicate mailfrom lines in configuration file %s", file); + break; + } + conf->mailfrom = estrdup(flds[2]); + }else if(i == 3 && strcmp(flds[1], "mailto") == 0) { + if(conf->mailto){ + seterr(EAdmin, "duplicate mailto lines in configuration file %s", file); + break; + } + conf->mailto = estrdup(flds[2]); + }else{ + seterr(EAdmin, "illegal line '%s' in configuration file %s", line, file); + break; + } + free(line); + line = nil; + } + free(line); + freeifile(&f); + return ok; +} + +static QLock loglk; +static char *logbuf; + +char* +logtext(VtLog *l) +{ + int i; + char *p; + VtLogChunk *c; + + p = logbuf; + c = l->w; + for(i=0; inchunk; i++) { + if(++c == l->chunk+l->nchunk) + c = l->chunk; + memmove(p, c->p, c->wp - c->p); + p += c->wp - c->p; + } + *p = 0; + return logbuf; +} + + +typedef struct HttpObj HttpObj; + +static int fromwebdir(HConnect*); + +enum +{ + ObjNameSize = 64, + MaxObjs = 64 +}; + +struct HttpObj +{ + char name[ObjNameSize]; + int (*f)(HConnect*); +}; + +static HttpObj objs[MaxObjs]; +static void httpproc(void*); + +static HConnect* +mkconnect(void) +{ + HConnect *c; + + c = mallocz(sizeof(HConnect), 1); + if(c == nil) + sysfatal("out of memory"); + c->replog = nil; + c->hpos = c->header; + c->hstop = c->header; + return c; +} + +static int +preq(HConnect *c) +{ + if(hparseheaders(c, 0) < 0) + return -1; + if(strcmp(c->req.meth, "GET") != 0 + && strcmp(c->req.meth, "HEAD") != 0) + return hunallowed(c, "GET, HEAD"); + if(c->head.expectother || c->head.expectcont) + return hfail(c, HExpectFail, nil); + return 0; +} + +int +hsettype(HConnect *c, char *type) +{ + Hio *hout; + int r; + + r = preq(c); + if(r < 0) + return r; + + hout = &c->hout; + if(c->req.vermaj){ + hokheaders(c); + hprint(hout, "Content-type: %s\r\n", type); + if(http11(c)) + hprint(hout, "Transfer-Encoding: chunked\r\n"); + hprint(hout, "\r\n"); + } + + if(http11(c)) + hxferenc(hout, 1); + else + c->head.closeit = 1; + return 0; +} + +int +hsethtml(HConnect *c) +{ + return hsettype(c, "text/html; charset=utf-8"); +} + +int +hsettext(HConnect *c) +{ + return hsettype(c, "text/plain; charset=utf-8"); +} + +int +hnotfound(HConnect *c) +{ + int r; + + r = preq(c); + if(r < 0) + return r; + return hfail(c, HNotFound, c->req.uri); +} + +static int +xloglist(HConnect *c) +{ + if(hsettype(c, "text/html") < 0) + return -1; + vtloghlist(&c->hout); + hflush(&c->hout); + return 0; +} + +static int +strpcmp(const void *va, const void *vb) +{ + return strcmp(*(char**)va, *(char**)vb); +} + +void +vtloghlist(Hio *h) +{ + char **p; + int i, n; + + hprint(h, "\n"); + hprint(h, "Venti Server Logs\n"); + hprint(h, "\n"); + hprint(h, "Venti Server Logs\n

\n"); + + p = vtlognames(&n); + qsort(p, n, sizeof(p[0]), strpcmp); + for(i=0; i%s
\n", p[i], p[i]); + vtfree(p); + hprint(h, "\n"); +} + +void +vtloghdump(Hio *h, VtLog *l) +{ + int i; + VtLogChunk *c; + char *name; + + name = l ? l->name : "<nil>"; + + hprint(h, "\n"); + hprint(h, "Venti Server Log: %s\n", name); + hprint(h, "\n"); + hprint(h, "Venti Server Log: %s\n

\n", name); + + if(l){ + c = l->w; + for(i=0; inchunk; i++){ + if(++c == l->chunk+l->nchunk) + c = l->chunk; + hwrite(h, c->p, c->wp-c->p); + } + } + hprint(h, "\n"); +} + + +char* +hargstr(HConnect *c, char *name, char *def) +{ + HSPairs *p; + + for(p=c->req.searchpairs; p; p=p->next) + if(strcmp(p->s, name) == 0) + return p->t; + return def; +} + +static int +xlog(HConnect *c) +{ + char *name; + VtLog *l; + + name = hargstr(c, "log", ""); + if(!name[0]) + return xloglist(c); + l = vtlogopen(name, 0); + if(l == nil) + return hnotfound(c); + if(hsettype(c, "text/html") < 0){ + vtlogclose(l); + return -1; + } + vtloghdump(&c->hout, l); + vtlogclose(l); + hflush(&c->hout); + return 0; +} + +static void +httpdproc(void *vaddress) +{ + HConnect *c; + char *address, ndir[NETPATHLEN], dir[NETPATHLEN]; + int ctl, nctl, data; + + address = vaddress; + ctl = announce(address, dir); + if(ctl < 0){ + sysfatal("announce %s: %r", address); + return; + } + + if(0) print("announce ctl %d dir %s\n", ctl, dir); + for(;;){ + /* + * wait for a call (or an error) + */ + nctl = listen(dir, ndir); + if(0) print("httpd listen %d %s...\n", nctl, ndir); + if(nctl < 0){ + fprint(2, "mgr: httpd can't listen on %s: %r\n", address); + return; + } + + data = accept(ctl, ndir); + if(0) print("httpd accept %d...\n", data); + if(data < 0){ + fprint(2, "mgr: httpd accept: %r\n"); + close(nctl); + continue; + } + if(0) print("httpd close nctl %d\n", nctl); + close(nctl); + c = mkconnect(); + hinit(&c->hin, data, Hread); + hinit(&c->hout, data, Hwrite); + vtproc(httpproc, c); + } +} + +static void +httpproc(void *v) +{ + HConnect *c; + int ok, i, n; + + c = v; + + for(;;){ + /* + * No timeout because the signal appears to hit every + * proc, not just us. + */ + if(hparsereq(c, 0) < 0) + break; + + for(i = 0; i < MaxObjs && objs[i].name[0]; i++){ + n = strlen(objs[i].name); + if((objs[i].name[n-1] == '/' && strncmp(c->req.uri, objs[i].name, n) == 0) + || (objs[i].name[n-1] != '/' && strcmp(c->req.uri, objs[i].name) == 0)){ + ok = (*objs[i].f)(c); + goto found; + } + } + ok = fromwebdir(c); + found: + hflush(&c->hout); + if(c->head.closeit) + ok = -1; + hreqcleanup(c); + + if(ok < 0) + break; + } + hreqcleanup(c); + close(c->hin.fd); + free(c); +} + +static int +httpdobj(char *name, int (*f)(HConnect*)) +{ + int i; + + if(name == nil || strlen(name) >= ObjNameSize) + return -1; + for(i = 0; i < MaxObjs; i++){ + if(objs[i].name[0] == '\0'){ + strcpy(objs[i].name, name); + objs[i].f = f; + return 0; + } + if(strcmp(objs[i].name, name) == 0) + return -1; + } + return -1; +} + + +struct { + char *ext; + char *type; +} exttab[] = { + ".html", "text/html", + ".txt", "text/plain", + ".xml", "text/xml", + ".png", "image/png", + ".gif", "image/gif", + 0 +}; + +static int +fromwebdir(HConnect *c) +{ + char buf[4096], *p, *ext, *type; + int i, fd, n, defaulted; + Dir *d; + + if(conf.webroot == nil || strstr(c->req.uri, "..")) + return hnotfound(c); + snprint(buf, sizeof buf-20, "%s/%s", conf.webroot, c->req.uri+1); + defaulted = 0; +reopen: + if((fd = open(buf, OREAD)) < 0) + return hnotfound(c); + d = dirfstat(fd); + if(d == nil){ + close(fd); + return hnotfound(c); + } + if(d->mode&DMDIR){ + if(!defaulted){ + defaulted = 1; + strcat(buf, "/index.html"); + free(d); + close(fd); + goto reopen; + } + free(d); + return hnotfound(c); + } + free(d); + p = buf+strlen(buf); + type = "application/octet-stream"; + for(i=0; exttab[i].ext; i++){ + ext = exttab[i].ext; + if(p-strlen(ext) >= buf && strcmp(p-strlen(ext), ext) == 0){ + type = exttab[i].type; + break; + } + } + if(hsettype(c, type) < 0){ + close(fd); + return 0; + } + while((n = read(fd, buf, sizeof buf)) > 0) + if(hwrite(&c->hout, buf, n) < 0) + break; + close(fd); + hflush(&c->hout); + return 0; +} + +static int +hmanager(HConnect *c) +{ + Hio *hout; + int r; + int i, k; + Job *j; + VtLog *l; + VtLogChunk *ch; + + r = hsethtml(c); + if(r < 0) + return r; + + hout = &c->hout; + hprint(hout, "venti mgr status\n"); + hprint(hout, "

venti mgr status

\n"); + + for(i=0; i"); + if(j->nrun == 0) + hprint(hout, "----/--/-- --:--:--"); + else + hprint(hout, "%+T", (long)(j->runstart + time0)); + hprint(hout, " %s", j->name); + if(j->nrun > 0) { + if(j->newok == -1) { + hprint(hout, " (running)"); + } else if(!j->newok) { + hprint(hout, " (FAILED)"); + } + } + hprint(hout, "\n"); + hprint(hout, "
\n");
+		l = j->newlog;
+		ch = l->w;
+		for(k=0; knchunk; k++){
+			if(++ch == l->chunk+l->nchunk)
+				ch = l->chunk;
+			hwrite(hout, ch->p, ch->wp-ch->p);
+		}
+		hprint(hout, "
\n"); + hprint(hout, "\n"); + } + hprint(hout, "\n"); + hflush(hout); + return 0; +} + +void +piper(void *v) +{ + Job *j; + char buf[512]; + VtLog *l; + int n; + int fd; + char *p; + int ok; + + j = v; + fd = j->pipe; + l = j->newlog; + while((n = read(fd, buf, 512-1)) > 0) { + buf[n] = 0; + if(l != nil) + vtlogprint(l, "%s", buf); + } + qlock(&loglk); + p = logtext(l); + ok = j->ok(p); + qunlock(&loglk); + j->newok = ok; + close(fd); +} + +void +kickjob(Job *j) +{ + int i; + int fd[3]; + int p[2]; + VtLog *l; + + if((fd[0] = open("/dev/null", ORDWR)) < 0) { + vtlogprint(errlog, "%T open /dev/null: %r\n"); + return; + } + if(pipe(p) < 0) { + vtlogprint(errlog, "%T pipe: %r\n"); + close(fd[0]); + return; + } + qlock(&j->lk); + l = j->oldlog; + j->oldlog = j->newlog; + j->newlog = l; + qlock(&l->lk); + for(i=0; inchunk; i++) + l->chunk[i].wp = l->chunk[i].p; + qunlock(&l->lk); + j->oldok = j->newok; + j->newok = -1; + qunlock(&j->lk); + + fd[1] = p[1]; + fd[2] = p[1]; + j->pid = threadspawn(fd, j->argv[0], j->argv); + if(j->pid < 0) { + vtlogprint(errlog, "%T exec %s: %r\n", j->argv[0]); + close(fd[0]); + close(fd[1]); + close(p[0]); + } + // fd[0], fd[1], fd[2] are closed now + j->pipe = p[0]; + j->nrun++; + vtproc(piper, j); +} + +int +getline(Resub *text, Resub *line) +{ + char *p; + + if(text->sp >= text->ep) + return -1; + line->sp = text->sp; + p = memchr(text->sp, '\n', text->ep - text->sp); + if(p == nil) { + line->ep = text->ep; + text->sp = text->ep; + } else { + line->ep = p; + text->sp = p+1; + } + return 0; +} + +int +verifyok(char *output) +{ + Resub text, line, m; + + text.sp = output; + text.ep = output+strlen(output); + while(getline(&text, &line) >= 0) { + *line.ep = 0; + memset(&m, 0, sizeof m); + if(!regexec(verifyprog, line.sp, nil, 0)) + return 0; + *line.ep = '\n'; + } + return 1; +} + +int +mirrorok(char *output) +{ + Resub text, line, m; + + text.sp = output; + text.ep = output+strlen(output); + while(getline(&text, &line) >= 0) { + *line.ep = 0; + memset(&m, 0, sizeof m); + if(!regexec(mirrorprog, line.sp, nil, 0)) + return 0; + *line.ep = '\n'; + } + return 1; +} + +void +mkjob(Job *j, ...) +{ + int i; + char *p; + va_list arg; + + memset(j, 0, sizeof *j); + i = 0; + va_start(arg, j); + while((p = va_arg(arg, char*)) != nil) { + j->argv[i++] = p; + if(i >= nelem(j->argv)) + sysfatal("job argv size too small"); + } + j->argv[i] = nil; + j->oldlog = vtlogopen(smprint("log%ld.0", j-job), LogSize); + j->newlog = vtlogopen(smprint("log%ld.1", j-job), LogSize); + va_end(arg); +} + +void +manager(void *v) +{ + int i; + Job *j; + vlong now; + + USED(v); + for(;; sleep(1000)) { + for(i=0; ipid > 0 || j->newok == -1) { + // still running + if(now - j->runstart > 2*j->freq) { + //TODO: log slow running j + } + continue; + } + if((j->nrun > 0 && now - j->runend > j->freq) + || (j->nrun == 0 && now > (vlong)(j->offset*j->freq))) { + j->runstart = now; + j->runend = 0; + kickjob(j); + } + } + } +} + +void +waitproc(void *v) +{ + Channel *c; + Waitmsg *w; + int i; + Job *j; + + c = v; + for(;;) { + w = recvp(c); + for(i=0; ipid == w->pid) { + j->pid = 0; + j->runend = time(0) - time0; + break; + } + } + free(w); + } +} + +void +threadmain(int argc, char **argv) +{ + int i; + int nofork; + char *prog; + Job *j; + + ventilogging = 1; + ventifmtinstall(); +#ifdef PLAN9PORT + bin = unsharp("#9/bin/venti"); +#else + bin = "/bin/venti"; +#endif + nofork = 0; + ARGBEGIN{ + case 'b': + bin = EARGF(usage()); + break; + case 's': + nofork = 1; + break; + default: + usage(); + }ARGEND + + if(argc != 1) + usage(); + if(rdconf(argv[0], &conf) < 0) + sysfatal("reading config: %r"); + if(conf.httpaddr == nil) + sysfatal("config has no httpaddr"); + if(conf.smtp != nil && conf.mailfrom == nil) + sysfatal("config has smtp but no mailfrom"); + if(conf.smtp != nil && conf.mailto == nil) + sysfatal("config has smtp but no mailto"); + if((mirrorprog = regcomp(mirrorregexp)) == nil) + sysfatal("mirrorregexp did not complete"); + if((verifyprog = regcomp(verifyregexp)) == nil) + sysfatal("verifyregexp did not complete"); + if(conf.nverify > 0 && conf.verifyfreq == 0) + sysfatal("config has no verifyfreq"); + if(conf.nmirror > 0 && conf.mirrorfreq == 0) + sysfatal("config has no mirrorfreq"); + + time0 = time(0); +// sendmail("startup", "mgr is starting\n"); + + logbuf = vtmalloc(LogSize+1); // +1 for NUL + + errlog = vtlogopen("errors", LogSize); + job = vtmalloc((conf.nmirror+conf.nverify)*sizeof job[0]); + prog = smprint("%s/mirrorarenas", bin); + for(i=0; iname = smprint("mirror %s %s", conf.mirror[i].src, conf.mirror[i].dst); + j->ok = mirrorok; + j->freq = conf.mirrorfreq; // 4 hours // TODO: put in config + j->offset = (double)i/conf.nmirror; + } + + prog = smprint("%s/verifyarena", bin); + for(i=0; iname = smprint("verify %s", conf.verify[i]); + j->ok = verifyok; + j->freq = conf.verifyfreq; + j->offset = (double)i/conf.nverify; + } + + httpdobj("/mgr", hmanager); + httpdobj("/log", xlog); + vtproc(httpdproc, conf.httpaddr); + vtproc(waitproc, threadwaitchan()); + if(nofork) + manager(nil); + else + vtproc(manager, nil); +} + + +void +qp(Biobuf *b, char *p) +{ + int n, nspace; + + nspace = 0; + n = 0; + for(; *p; p++) { + if(*p == '\n') { + if(nspace > 0) { + nspace = 0; + Bprint(b, "=\n"); + } + Bputc(b, '\n'); + n = 0; + continue; + } + if(n > 70) { + Bprint(b, "=\n"); + nspace = 0; + continue; + } + if(33 <= *p && *p <= 126 && *p != '=') { + Bputc(b, *p); + n++; + nspace = 0; + continue; + } + if(*p == ' ' || *p == '\t') { + Bputc(b, *p); + n++; + nspace++; + continue; + } + Bprint(b, "=%02X", (uchar)*p); + n += 3; + nspace = 0; + } +} + +int +smtpread(Biobuf *b, int code) +{ + char *p, *q; + int n; + + while((p = Brdstr(b, '\n', 1)) != nil) { + n = strtol(p, &q, 10); + if(n == 0 || q != p+3) { + error: + vtlogprint(errlog, "sending mail: %s\n", p); + free(p); + return -1; + } + if(*q == ' ') { + if(n == code) { + free(p); + return 0; + } + goto error; + } + if(*q != '-') { + goto error; + } + } + return -1; +} + + +void +sendmail(char *content, char *subject, char *msg) +{ + int fd; + Biobuf *bin, *bout; + + if((fd = dial(conf.smtp, 0, 0, 0)) < 0) { + vtlogprint(errlog, "dial %s: %r\n", conf.smtp); + return; + } + bin = vtmalloc(sizeof *bin); + bout = vtmalloc(sizeof *bout); + Binit(bin, fd, OREAD); + Binit(bout, fd, OWRITE); + if(smtpread(bin, 220) < 0){ + error: + close(fd); + Bterm(bin); + Bterm(bout); + return; + } + + Bprint(bout, "HELO venti-mgr\n"); + Bflush(bout); + if(smtpread(bin, 250) < 0) + goto error; + + Bprint(bout, "MAIL FROM:<%s>\n", conf.mailfrom); + Bflush(bout); + if(smtpread(bin, 250) < 0) + goto error; + + Bprint(bout, "RCPT TO:<%s>\n", conf.mailfrom); + Bflush(bout); + if(smtpread(bin, 250) < 0) + goto error; + + Bprint(bout, "DATA\n"); + Bflush(bout); + if(smtpread(bin, 354) < 0) + goto error; + + Bprint(bout, "From: \"venti mgr\" <%s>\n", conf.mailfrom); + Bprint(bout, "To: <%s>\n", conf.mailto); + Bprint(bout, "Subject: %s\n", subject); + Bprint(bout, "MIME-Version: 1.0\n"); + Bprint(bout, "Content-Type: %s; charset=\"UTF-8\"\n", content); + Bprint(bout, "Content-Transfer-Encoding: quoted-printable\n"); + Bprint(bout, "Message-ID: %08lux%08lux@venti.swtch.com\n", fastrand(), fastrand()); + Bprint(bout, "\n"); + qp(bout, msg); + Bprint(bout, ".\n"); + Bflush(bout); + if(smtpread(bin, 250) < 0) + goto error; + + Bprint(bout, "QUIT\n"); + Bflush(bout); + Bterm(bin); + Bterm(bout); + close(fd); +} diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/mirror-log.awk --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/src/cmd/venti/srv/mirror-log.awk Tue May 29 00:00:00 2012 +0200 @@ -0,0 +1,105 @@ +# possible cron job: +# +# cd /cfg/backup +# { for(i in 'E0 E1' 'E2 E3' 'E4 E5' 'E6 E7' 'F0 F1' 'F2 F3' 'F4 F5'){ +# x=`{echo $i} +# venti/mirrorarenas -v /dev/sd$x(1)^/arenas /dev/sd$x(2)^/arenas +# } } >www/mirror1.txt >[2=1] +# mv www/mirror1.txt www/mirror.txt +# awk -f mirror-log.awk www/mirror.txt >www/mirror.html + +BEGIN { + print "

mirror status

" + print "details in mirror.txt

" + print "
" + laststatus = "" + firstarena = "" + lastarena = "" + status = "" + arena = "" + +} + +function fmt( color) { + nfmt++ + if(nfmt%2 == 0) + color = "#cccccc" + else + color = "#ffffff" + return "" + firstarena = "" + lastarena = "" + laststatus = "" + } + if(info == ""){ + firstarena = arena + laststatus = status + lastarena = arena + return + } + printf(fmt(), time, arena, "", ""); + print status + if(info != ""){ + print "
"
+		printf("%s", info)
+		print "
" + } + print "" +} + +$3 !~ /:$/ && $4 ~ /^\(.*-.*\)$/ { + finish(); + arena = $3 + range = $4 + status = "" + info = "" + size = 0 + time = $1 " " $2 + next +} + +$3 ~ /:$/ && $0 ~ /^....\/.... ..:..:.. [^ ]/ { + if($4 == "0" && $5 == "used" && $6 == "mirrored"){ + status = "empty" + next + } + if($4 ~ /^[0-9,]+$/ && $5 == "used" && $6 == "mirrored"){ + size = $4 + status = "partial " size ", mirrored" + next + } + if($4 ~ /^[0-9a-f]+$/ && length($4) == 40 && $5 == "sealed" && $6 == "mirrored"){ + status = "sealed, mirrored"; + next + } +} + +{ + info = info $0 "\n" +} + +END{ + finish(); + status = "done" + arena = "" + info = "" + finish(); + print "
%s%s%s%s" +} + + +function finish() { + if(!arena && !status) + return + if(info == "" && laststatus == status){ + lastarena = arena + return + } + if(firstarena != ""){ + if(firstarena == lastarena) + printf(fmt(), time, firstarena, "", ""); + else + printf(fmt(), time, firstarena, "-", lastarena); + print laststatus "
" + print "" +} diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/mirrorarenas.c --- a/sys/src/cmd/venti/srv/mirrorarenas.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/mirrorarenas.c Tue May 29 00:00:00 2012 +0200 @@ -34,9 +34,11 @@ } char *tagged; +char *tagname; +int tagindx; void -tag(char *fmt, ...) +tag(int indx, char *name, char *fmt, ...) { va_list arg; @@ -44,16 +46,72 @@ free(tagged); tagged = nil; } + tagindx = indx; + tagname = name; va_start(arg, fmt); tagged = vsmprint(fmt, arg); va_end(arg); } +enum +{ + Sealed = 1, + Mirrored = 2, + Empty = 4, +}; + +void +setstatus(int bits) +{ + static int startindx = -1, endindx; + static char *startname, *endname; + static int lastbits; + char buf[100]; + + if(bits != lastbits) { + if(startindx >= 0) { + switch(lastbits) { + case Sealed: + snprint(buf, sizeof buf, "sealed"); + break; + case Mirrored: + snprint(buf, sizeof buf, "mirrored"); + break; + case Sealed+Mirrored: + snprint(buf, sizeof buf, "mirrored sealed"); + break; + case Empty: + snprint(buf, sizeof buf, "empty"); + break; + default: + snprint(buf, sizeof buf, "%d", bits); + break; + } + print("%T %s-%s %s\n", startname, endname, buf); + } + lastbits = bits; + startindx = tagindx; + endindx = tagindx; + startname = tagname; + endname = tagname; + } else { + endindx = tagindx; + endname = tagname; + } + if(bits < 0) { + startindx = -1; + endindx = -1; + return; + } +} + void chat(char *fmt, ...) { va_list arg; + setstatus(-1); + if(tagged){ write(1, tagged, strlen(tagged)); free(tagged); @@ -64,7 +122,7 @@ va_end(arg); } -#pragma varargck argpos tag 1 +#pragma varargck argpos tag 3 #pragma varargck argpos chat 1 @@ -113,13 +171,25 @@ { int i, n; uvlong o; - static uchar tmp[2][1024*1024]; + enum { + Chunk = 1024*1024 + }; + static uchar tmpbuf[2*Chunk+MaxIo]; + static uchar *tmp[2]; + uchar *p; Write w[2]; assert(start <= end); assert(astart <= start && start < aend); assert(astart <= end && end <= aend); + // align the buffers so readpart/writepart can do big transfers + p = tmpbuf; + if((uintptr)p%MaxIo) + p += MaxIo - (uintptr)p%MaxIo; + tmp[0] = p; + tmp[1] = p + Chunk; + if(verbose && start != end) chat("%T copy %,llud-%,llud %s\n", start, end, what); @@ -128,7 +198,7 @@ for(o=start; o end) n = end - o; if(ereadpart(src, o, tmp[i], n) < 0) @@ -235,7 +305,7 @@ } void -mirror(Arena *sa, Arena *da) +mirror(int indx, Arena *sa, Arena *da) { vlong v, si, di, end; int clumpmax, blocksize, sealed; @@ -251,7 +321,7 @@ astart = base - blocksize; aend = end + blocksize; - tag("%T %s (%,llud-%,llud)\n", sa->name, astart, aend); + tag(indx, sa->name, "%T %s (%,llud-%,llud)\n", sa->name, astart, aend); if(force){ copy(astart, aend, "all", nil); @@ -260,7 +330,8 @@ if(sa->diskstats.sealed && da->diskstats.sealed && scorecmp(da->score, zeroscore) != 0){ if(scorecmp(sa->score, da->score) == 0){ - if(verbose) + setstatus(Sealed+Mirrored); + if(verbose > 1) chat("%T %s: %V sealed mirrored\n", sa->name, sa->score); return; } @@ -378,7 +449,8 @@ memset(buf, 0, VtScoreSize); sha1(buf, VtScoreSize, da->score, ds); if(scorecmp(sa->score, da->score) == 0){ - if(verbose) + setstatus(Sealed+Mirrored); + if(verbose > 1) chat("%T %s: %V sealed mirrored\n", sa->name, sa->score); if(ewritepart(dst, end+blocksize-VtScoreSize, da->score, VtScoreSize) < 0) return; @@ -391,14 +463,21 @@ status = "errors"; } }else{ - if(verbose) + setstatus(Mirrored); + if(verbose > 1) chat("%T %s: %V mirrored\n", sa->name, sa->score); if(ewritepart(dst, end+blocksize-VtScoreSize, sa->score, VtScoreSize) < 0) return; } }else{ - chat("%T %s: %,lld used mirrored\n", - sa->name, sa->diskstats.used); + if(sa->diskstats.used > 0 || verbose > 1) { + chat("%T %s: %,lld used mirrored\n", + sa->name, sa->diskstats.used); + } + if(sa->diskstats.used > 0) + setstatus(Mirrored); + else + setstatus(Empty); } } @@ -413,8 +492,9 @@ for(i=0; inarenas; i++){ sa = sp->arenas[i]; da = dp->arenas[i]; - mirror(sa, da); + mirror(i, sa, da); } + setstatus(-1); return; } if(strcmp(range, "none") == 0) @@ -445,8 +525,9 @@ for(i=lo; i<=hi; i++){ sa = sp->arenas[i]; da = dp->arenas[i]; - mirror(sa, da); + mirror(i, sa, da); } + setstatus(-1); } } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/printarenapart.c --- a/sys/src/cmd/venti/srv/printarenapart.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/printarenapart.c Tue May 29 00:00:00 2012 +0200 @@ -11,7 +11,8 @@ threadexitsall("usage"); } -static void +/* unused */ +void rdarena(Arena *arena, u64int offset) { u64int a, aa, e; diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/rdarena.c --- a/sys/src/cmd/venti/srv/rdarena.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/rdarena.c Tue May 29 00:00:00 2012 +0200 @@ -7,7 +7,7 @@ void usage(void) { - fprint(2, "usage: rdarena [-v] arenapart arena\n"); + fprint(2, "usage: rdarena [-qv] arenapart arena\n"); threadexitsall(0); } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/venti.c --- a/sys/src/cmd/venti/srv/venti.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/venti.c Tue May 29 00:00:00 2012 +0200 @@ -149,9 +149,8 @@ void usage(void) { - fprint(2, "usage: venti [-Ldrsw] [-a ventiaddr] [-c config] " -"[-h httpaddr] [-m %%mem] [-B blockcachesize] [-C cachesize] [-I icachesize] " -"[-W webroot]\n"); + fprint(2, "usage: venti [-Ldrs] [-a address] [-B blockcachesize] [-c config] " +"[-C lumpcachesize] [-h httpaddress] [-I indexcachesize] [-m %%mem] [-W webroot]\n"); threadexitsall("usage"); } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/verifyarena.c --- a/sys/src/cmd/venti/srv/verifyarena.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/verifyarena.c Tue May 29 00:00:00 2012 +0200 @@ -60,7 +60,8 @@ u32int bs; u8int score[VtScoreSize]; - fprint(2, "%T verify %s\n", name); + if(verbose) + fprint(2, "%T verify %s\n", name); memset(&arena, 0, sizeof arena); memset(&s, 0, sizeof s); @@ -140,16 +141,19 @@ /* * check for no checksum or the same */ - if(scorecmp(score, arena.score) == 0) - fprint(2, "%T %s: verified score\n", name); - else if(scorecmp(zeroscore, arena.score) == 0) - fprint(2, "%T %s: unsealed\n", name); - else{ + if(scorecmp(score, arena.score) == 0) { + if(verbose) + fprint(2, "%T %s: verified score\n", name); + } else if(scorecmp(zeroscore, arena.score) == 0) { + if(verbose || arena.diskstats.used > 0) + fprint(2, "%T %s: unsealed %,lld bytes\n", name, arena.diskstats.used); + } else{ fprint(2, "%T %s: mismatch checksum - found=%V calculated=%V\n", name, arena.score, score); return; } - printarena(2, &arena); + if(verbose > 1) + printarena(2, &arena); } static int @@ -196,7 +200,10 @@ break; }ARGEND - data = vtmalloc(blocksize); + data = vtmalloc(MaxIo + blocksize); + if((uintptr)data % MaxIo) + data += MaxIo - (uintptr)data%MaxIo; + if(argc == 0){ fd = 0; verifyarena("", 0); @@ -212,8 +219,9 @@ sysfatal("read arena part header: %r"); if(unpackarenapart(&ap, data) < 0) sysfatal("corrupted arena part header: %r"); - fprint(2, "%T # arena part version=%d blocksize=%d arenabase=%d\n", - ap.version, ap.blocksize, ap.arenabase); + if(verbose) + fprint(2, "%T # arena part version=%d blocksize=%d arenabase=%d\n", + ap.version, ap.blocksize, ap.arenabase); ap.tabbase = (PartBlank+HeadSize+ap.blocksize-1)&~(ap.blocksize-1); ap.tabsize = ap.arenabase - ap.tabbase; table = malloc(ap.tabsize+1); diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/wrarena.c --- a/sys/src/cmd/venti/srv/wrarena.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/wrarena.c Tue May 29 00:00:00 2012 +0200 @@ -24,7 +24,7 @@ void usage(void) { - fprint(2, "usage: wrarena [-h host] arenafile [offset]\n"); + fprint(2, "usage: wrarena [-o fileoffset] [-h host] arenafile [clumpoffset]\n"); threadexitsall("usage"); } @@ -131,13 +131,13 @@ int i; char *file; Arena *arena; + ArenaPart *ap; u64int offset, aoffset; Part *part; uchar buf[8192]; ArenaHead head; ZClump zerocl; - ventifmtinstall(); qlock(&godot); aoffset = 0; ARGBEGIN{ @@ -174,13 +174,21 @@ file = argv[0]; } - fmtinstall('V', vtscorefmt); + ventifmtinstall(); statsinit(); part = initpart(file, OREAD); if(part == nil) sysfatal("can't open file %s: %r", file); + initdcache(8 * MaxDiskBlock); + + // Try as arena partition. + arena = nil; + ap = initarenapart(part); + if(ap == nil) + goto loaded; + if(readpart(part, aoffset, buf, sizeof buf) < 0) sysfatal("can't read file %s: %r", file); @@ -192,12 +200,12 @@ head.size, part->size); partblocksize(part, head.blocksize); - initdcache(8 * MaxDiskBlock); arena = initarena(part, aoffset, head.size, head.blocksize); if(arena == nil) sysfatal("initarena: %r"); +loaded: z = nil; if(host==nil || strcmp(host, "/dev/null") != 0){ z = vtdial(host); @@ -207,19 +215,26 @@ sysfatal("vtconnect: %r"); } + print("%T starting to send data\n"); c = chancreate(sizeof(ZClump), 0); for(i=0; i<12; i++) vtproc(vtsendthread, nil); - rdarena(arena, offset); - if(vtsync(z) < 0) - sysfatal("executing sync: %r"); + if(ap != nil) { + for(i=0; inarenas; i++) + rdarena(ap->arenas[i], 0); + } else + rdarena(arena, offset); memset(&zerocl, 0, sizeof zerocl); for(i=0; i<12; i++) send(c, &zerocl); + if(vtsync(z) < 0) + sysfatal("executing sync: %r"); if(z){ vthangup(z); } + print("%T sent all data\n"); + threadexitsall(0); } diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/www/debug.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/src/cmd/venti/srv/www/debug.html Tue May 29 00:00:00 2012 +0200 @@ -0,0 +1,41 @@ +

venti debugging

+ +amap
+mem
+disk
+
+arena part: + + + +

+
+bloom filter: + + + +

+
+index section: + + + +

+
+
+ +read score: + + +

+
+
+read score: + + +in arena + +in arena partition + + +

diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/www/stats.html --- a/sys/src/cmd/venti/srv/www/stats.html Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/www/stats.html Tue May 29 00:00:00 2012 +0200 @@ -6,6 +6,7 @@ +
venti.your-domain.com – venti server statistics diff -r 88ea8de5bdf7 sys/src/cmd/venti/srv/www/stats.js --- a/sys/src/cmd/venti/srv/www/stats.js Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/srv/www/stats.js Tue May 29 00:00:00 2012 +0200 @@ -15,11 +15,11 @@ "arena write bytes/second", "arg=bloomfalsemiss&graph=pctdiff&arg2=bloomlookup&max=100", - "bloom false hit %", + "bloom false miss %", "arg=bloomhit&graph=pctdiff&arg2=bloomlookup&max=100", + "bloom hit %", + "arg=bloommiss&graph=pctdiff&arg2=bloomlookup&max=100", "bloom miss %", - "arg=bloomlookuptime&graph=divdiff&arg2=bloomlookup", - "bloom lookup time", "arg=bloomones&graph=pct&arg2=bloombits&max=100", "bloom usage %", @@ -38,10 +38,16 @@ "icache dirty %", "arg=icachehit&graph=pctdiff&arg2=icachelookup&max=100", "icache hit %", + "arg=scachehit&graph=pctdiff&arg2=icachelookup&max=100", + "scache hit %", + "arg=icachemiss&graph=pctdiff&arg2=icachelookup&max=100", + "icache miss %", "arg=icachelookuptime&graph=divdiff&arg2=icachelookup", "icache lookup time", "arg=icacheprefetch&graph=diff", "icache prefetches/second", + "arg=scacheprefetch&graph=diff", + "scache prefetches/second", "arg=icachewrite&graph=diff", "icache writes/second", @@ -75,6 +81,8 @@ "fresh write RPC time", "arg=rpcwriteoldtime&graph=divdiff&arg2=rpcwriteold", "dup write RPC time", + "arg=cigloadtime&graph=divdiff&arg2=cigload", + "cig load time", "arg=sumreadbyte&graph=diff", "checksum bytes/second", @@ -107,6 +115,7 @@ "!bloom filter", "arg=bloomhit&graph=pctdiff&arg2=bloomlookup&max=100", + "arg=bloommiss&graph=pctdiff&arg2=bloomlookup&max=100", "arg=bloomfalsemiss&graph=pctdiff&arg2=bloomlookup&max=100", "arg=bloomones&graph=pct&arg2=bloombits&max=100", @@ -118,8 +127,11 @@ "!icache", "arg=icachedirty&graph=pct&arg2=icachesize&max=100", "arg=icachehit&graph=pctdiff&arg2=icachelookup&max=100", + "arg=scachehit&graph=pctdiff&arg2=icachelookup&max=100", + "arg=icachemiss&graph=pctdiff&arg2=icachelookup&max=100", "arg=icachewrite&graph=diff", "arg=icacheprefetch&graph=diff", + "arg=scacheprefetch&graph=diff", "!dcache", "arg=dcachedirty&graph=pct&arg2=dcachesize&max=100", @@ -144,7 +156,6 @@ "arg=lumpstall", "!timings", - "arg=bloomlookuptime&graph=divdiff&arg2=bloomlookup", "arg=icachelookuptime&graph=divdiff&arg2=icachelookup", "arg=lcachelookuptime&graph=divdiff&arg2=lcachelookup", "arg=dcachelookuptime&graph=divdiff&arg2=dcachelookup", @@ -154,6 +165,7 @@ "arg=rpcreaduncachedtime&graph=divdiff&arg2=rpcreaduncached", "arg=rpcwritenewtime&graph=divdiff&arg2=rpcwritenew", "arg=rpcwriteoldtime&graph=divdiff&arg2=rpcwriteold", + "arg=cigloadtime&graph=divdiff&arg2=cigload", "END" ) @@ -353,7 +365,7 @@ function loglinks(list) { var s = "" for(var i=0; i"+list[i]+"" + s = s+" "+list[i]+"" } return s } @@ -383,5 +395,5 @@ eval(name+"= \""+value+"\"") redrawsettings() // Works in FireFox, not in Safari - parent.hidden.location.href = "/set/"+name+"/"+value + parent.hidden.location.href = "/set?name="+name+"&value="+value } diff -r 88ea8de5bdf7 sys/src/cmd/venti/write.c --- a/sys/src/cmd/venti/write.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/cmd/venti/write.c Tue May 29 00:00:00 2012 +0200 @@ -4,6 +4,12 @@ #include #include +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65535, +}; + void usage(void) { diff -r 88ea8de5bdf7 sys/src/cmd/venti/writefile.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/src/cmd/venti/writefile.c Tue May 29 00:00:00 2012 +0200 @@ -0,0 +1,106 @@ +#include +#include +#include +#include +#include + +enum +{ + Blocksize = 8192 +}; + +int chatty; + +void +usage(void) +{ + fprint(2, "usage: writefile [-v] [-h host] < data\n"); + threadexitsall("usage"); +} + +void +threadmain(int argc, char *argv[]) +{ + int n; + uchar score[VtScoreSize]; + uchar *buf; + char *host; + vlong off; + VtEntry e; + VtRoot root; + VtCache *c; + VtConn *z; + VtFile *f; + + quotefmtinstall(); + fmtinstall('F', vtfcallfmt); + fmtinstall('V', vtscorefmt); + + host = nil; + ARGBEGIN{ + case 'V': + chattyventi++; + break; + case 'h': + host = EARGF(usage()); + break; + case 'v': + chatty++; + break; + default: + usage(); + break; + }ARGEND + + if(argc != 0) + usage(); + + buf = vtmallocz(Blocksize); + + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + // write file + c = vtcachealloc(z, Blocksize*32); + if(c == nil) + sysfatal("vtcachealloc: %r"); + f = vtfilecreateroot(c, Blocksize, Blocksize, VtDataType); + if(f == nil) + sysfatal("vtfilecreateroot: %r"); + off = 0; + vtfilelock(f, VtOWRITE); + while((n = read(0, buf, Blocksize)) > 0){ + if(vtfilewrite(f, buf, n, off) != n) + sysfatal("vtfilewrite: %r"); + off += n; + if(vtfileflushbefore(f, off) < 0) + sysfatal("vtfileflushbefore: %r"); + } + if(vtfileflush(f) < 0) + sysfatal("vtfileflush: %r"); + if(vtfilegetentry(f, &e) < 0) + sysfatal("vtfilegetentry: %r"); + vtfileunlock(f); + + // write directory entry + memset(&root, 0, sizeof root); + vtentrypack(&e, buf, 0); + if(vtwrite(z, root.score, VtDirType, buf, VtEntrySize) < 0) + sysfatal("vtwrite dir: %r"); + + // write root + strcpy(root.name, "data"); + strcpy(root.type, "file"); + root.blocksize = Blocksize; + vtrootpack(&root, buf); + if(vtwrite(z, score, VtRootType, buf, VtRootSize) < 0) + sysfatal("vtwrite root: %r"); + + print("file:%V\n", score); + threadexitsall(0); +} + diff -r 88ea8de5bdf7 sys/src/libventi/cache.c --- a/sys/src/libventi/cache.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/cache.c Tue May 29 00:00:00 2012 +0200 @@ -32,7 +32,6 @@ { QLock lk; VtConn *z; - u32int blocksize; u32int now; /* ticks for usage time stamps */ VtBlock **hash; /* hash table for finding addresses */ int nhash; @@ -40,41 +39,45 @@ int nheap; VtBlock *block; /* all allocated blocks */ int nblock; - uchar *mem; /* memory for all blocks and data */ int (*write)(VtConn*, uchar[VtScoreSize], uint, uchar*, int); + VtBlock *dead; /* blocks we don't have memory for */ + ulong mem; + ulong maxmem; }; static void cachecheck(VtCache*); VtCache* -vtcachealloc(VtConn *z, int blocksize, ulong nblock) +vtcachealloc(VtConn *z, ulong maxmem) { - uchar *p; VtCache *c; int i; + int nblock; VtBlock *b; + ulong maxmem0; + maxmem0 = maxmem; c = vtmallocz(sizeof(VtCache)); - + nblock = maxmem/100/(sizeof(VtBlock)+2*sizeof(VtBlock*)); c->z = z; - c->blocksize = (blocksize + 127) & ~127; c->nblock = nblock; c->nhash = nblock; c->hash = vtmallocz(nblock*sizeof(VtBlock*)); c->heap = vtmallocz(nblock*sizeof(VtBlock*)); c->block = vtmallocz(nblock*sizeof(VtBlock)); - c->mem = vtmallocz(nblock*c->blocksize); c->write = vtwrite; + maxmem -= nblock*(sizeof(VtBlock) + 2*sizeof(VtBlock*)); + maxmem -= sizeof(VtCache); + if((long)maxmem < 0) + sysfatal("cache size far too small: %lud", maxmem0); + c->mem = maxmem; - p = c->mem; for(i=0; iblock[i]; b->addr = NilBlock; b->c = c; - b->data = p; b->heap = i; c->heap[i] = b; - p += c->blocksize; } c->nheap = nblock; cachecheck(c); @@ -102,13 +105,14 @@ qlock(&c->lk); cachecheck(c); - for(i=0; inblock; i++) - assert(c->block[i].ref == 0); + for(i=0; inblock; i++) { + assert(c->block[i].data == nil || c->block[i].ref == 0); + vtfree(c->block[i].data); + } vtfree(c->hash); vtfree(c->heap); vtfree(c->block); - vtfree(c->mem); vtfree(c); } @@ -128,11 +132,10 @@ static void cachecheck(VtCache *c) { - u32int size, now; + u32int now; int i, k, refed; VtBlock *b; - size = c->blocksize; now = c->now; for(i = 0; i < c->nheap; i++){ @@ -151,8 +154,6 @@ refed = 0; for(i = 0; i < c->nblock; i++){ b = &c->block[i]; - if(b->data != &c->mem[i * size]) - sysfatal("mis-blocked at %d", i); if(b->ref && b->heap == BadHeap) refed++; else if(b->addr != NilBlock) @@ -299,6 +300,57 @@ } /* + * evict blocks until there is enough memory for size bytes. + */ +static VtBlock* +vtcacheevict(VtCache *c, ulong size) +{ + VtBlock *b; + + /* + * If we were out of memory and put some blocks + * to the side but now we have memory, grab one. + */ + if(c->mem >= size && c->dead) { + b = c->dead; + c->dead = b->next; + b->next = nil; + goto alloc; + } + + /* + * Otherwise, evict until we have memory. + */ + for(;;) { + b = vtcachebumpblock(c); + if(c->mem+b->size >= size) + break; + /* + * chain b onto dead list + */ + free(b->data); + b->data = nil; + c->mem += b->size; + b->size = 0; + b->next = c->dead; + c->dead = b; + } + + /* + * Allocate memory for block. + */ +alloc: + if(size > b->size || size <= b->size/2) { + free(b->data); + c->mem += b->size; + c->mem -= size; + b->size = size; + b->data = vtmalloc(size); + } + return b; +} + +/* * fetch a local block from the memory cache. * if it's not there, load it, bumping some other Block. * if we're out of free blocks, we're screwed. @@ -312,7 +364,7 @@ sysfatal("vtcachelocal: asked for nonexistent block 0"); if(addr > c->nblock) sysfatal("vtcachelocal: asked for block #%ud; only %d blocks", - addr, c->nblock); + (uint)addr, c->nblock); b = &c->block[addr-1]; if(b->addr == NilBlock || b->iostate != BioLocal) @@ -332,16 +384,16 @@ } VtBlock* -vtcacheallocblock(VtCache *c, int type) +vtcacheallocblock(VtCache *c, int type, ulong size) { VtBlock *b; qlock(&c->lk); - b = vtcachebumpblock(c); + b = vtcacheevict(c, size); b->iostate = BioLocal; b->type = type; b->addr = (b - c->block)+1; - vtzeroextend(type, b->data, 0, c->blocksize); + vtzeroextend(type, b->data, 0, size); vtlocaltoglobal(b->addr, b->score); qunlock(&c->lk); @@ -356,7 +408,7 @@ * if it's not there, load it, bumping some other block. */ VtBlock* -vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type) +vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type, ulong size) { VtBlock *b; ulong h; @@ -409,7 +461,7 @@ /* * not found */ - b = vtcachebumpblock(c); + b = vtcacheevict(c, size); b->addr = NilBlock; b->type = type; memmove(b->score, score, VtScoreSize); @@ -435,7 +487,7 @@ qunlock(&c->lk); vtcachenread++; - n = vtread(c->z, score, type, b->data, c->blocksize); + n = vtread(c->z, score, type, b->data, size); if(n < 0){ if(chattyventi) fprint(2, "read %V: %r\n", score); @@ -445,7 +497,7 @@ vtblockput(b); return nil; } - vtzeroextend(type, b->data, n, c->blocksize); + vtzeroextend(type, b->data, n, size); b->iostate = BioVenti; b->nlock = 1; if(vttracelevel) @@ -534,7 +586,7 @@ } c = b->c; - n = vtzerotruncate(b->type, b->data, c->blocksize); + n = vtzerotruncate(b->type, b->data, b->size); vtcachenwrite++; if(c->write(c->z, score, b->type, b->data, n) < 0) return -1; @@ -554,24 +606,18 @@ return 0; } -uint -vtcacheblocksize(VtCache *c) -{ - return c->blocksize; -} - VtBlock* vtblockcopy(VtBlock *b) { VtBlock *bb; vtcachencopy++; - bb = vtcacheallocblock(b->c, b->type); + bb = vtcacheallocblock(b->c, b->type, b->size); if(bb == nil){ vtblockput(b); return nil; } - memmove(bb->data, b->data, b->c->blocksize); + memmove(bb->data, b->data, b->size); vtblockput(b); bb->pc = getcallerpc(&b); return bb; diff -r 88ea8de5bdf7 sys/src/libventi/client.c --- a/sys/src/libventi/client.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/client.c Tue May 29 00:00:00 2012 +0200 @@ -65,6 +65,15 @@ if(memcmp(score, vtzeroscore, VtScoreSize) == 0) return packetalloc(); + if(z == nil){ + werrstr("not connected"); + return nil; + } + + if(z->version[1] == '2' && n >= (1<<16)) { + werrstr("read count too large for protocol"); + return nil; + } memset(&tx, 0, sizeof tx); tx.msgtype = VtTread; tx.blocktype = type; diff -r 88ea8de5bdf7 sys/src/libventi/conn.c --- a/sys/src/libventi/conn.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/conn.c Tue May 29 00:00:00 2012 +0200 @@ -27,6 +27,28 @@ return z; } +int +vtreconn(VtConn *z, int infd, int outfd) +{ + NetConnInfo *nci; + + z->state = VtStateAlloc; + if(z->infd >= 0) + close(z->infd); + z->infd = infd; + if(z->outfd >= 0) + close(z->outfd); + z->outfd = outfd; + nci = getnetconninfo(nil, infd); + if(nci == nil) + snprint(z->addr, sizeof z->addr, "/dev/fd/%d", infd); + else{ + strecpy(z->addr, z->addr+sizeof z->addr, nci->raddr); + freenetconninfo(nci); + } + return 0; +} + void vtfreeconn(VtConn *z) { diff -r 88ea8de5bdf7 sys/src/libventi/dial.c --- a/sys/src/libventi/dial.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/dial.c Tue May 29 00:00:00 2012 +0200 @@ -23,3 +23,21 @@ strecpy(z->addr, z->addr+sizeof z->addr, na); return z; } + +int +vtredial(VtConn *z, char *addr) +{ + char *na; + int fd; + + if(addr == nil) + addr = getenv("venti"); + if(addr == nil) + addr = "$venti"; + + na = netmkaddr(addr, "tcp", "venti"); + if((fd = dial(na, nil, nil, nil)) < 0) + return fd; + + return vtreconn(z, fd, fd); +} diff -r 88ea8de5bdf7 sys/src/libventi/entry.c --- a/sys/src/libventi/entry.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/entry.c Tue May 29 00:00:00 2012 +0200 @@ -6,13 +6,37 @@ static int checksize(int n) { - if(n < 256 || n > VtMaxLumpSize) { + if(n < 256) { werrstr("bad block size %#ux", n); return -1; } return 0; } +// _VtEntryBig integer format is floating-point: +// (n>>5) << (n&31). +// Convert this number; must be exact or return -1. +int +vttobig(ulong n) +{ + int shift; + ulong n0; + + n0 = n; + shift = 0; + while(n >= (1<<(16 - 5))) { + if(n & 1) + return -1; + shift++; + n >>= 1; + } + + n = (n<<5) | shift; + if(((n>>5)<<(n&31)) != n0) + sysfatal("vttobig %#lux => %#lux failed", n0, n); + return n; +} + void vtentrypack(VtEntry *e, uchar *p, int index) { @@ -20,21 +44,31 @@ int flags; uchar *op; int depth; + int psize, dsize; p += index * VtEntrySize; op = p; - U32PUT(p, e->gen); - p += 4; - U16PUT(p, e->psize); - p += 2; - U16PUT(p, e->dsize); - p += 2; depth = e->type&VtTypeDepthMask; flags = (e->flags&~(_VtEntryDir|_VtEntryDepthMask)); flags |= depth << _VtEntryDepthShift; if(e->type - depth == VtDirType) flags |= _VtEntryDir; + U32PUT(p, e->gen); + p += 4; + psize = e->psize; + dsize = e->dsize; + if(psize >= (1<<16) || dsize >= (1<<16)) { + flags |= _VtEntryBig; + psize = vttobig(psize); + dsize = vttobig(dsize); + if(psize < 0 || dsize < 0) + sysfatal("invalid entry psize/dsize: %ld/%ld", e->psize, e->dsize); + } + U16PUT(p, psize); + p += 2; + U16PUT(p, dsize); + p += 2; U8PUT(p, flags); p++; memset(p, 0, 5); @@ -62,10 +96,14 @@ e->dsize = U16GET(p); p += 2; e->flags = U8GET(p); + p++; + if(e->flags & _VtEntryBig) { + e->psize = (e->psize>>5)<<(e->psize & 31); + e->dsize = (e->dsize>>5)<<(e->dsize & 31); + } e->type = (e->flags&_VtEntryDir) ? VtDirType : VtDataType; e->type += (e->flags & _VtEntryDepthMask) >> _VtEntryDepthShift; - e->flags &= ~(_VtEntryDir|_VtEntryDepthMask); - p++; + e->flags &= ~(_VtEntryDir|_VtEntryDepthMask|_VtEntryBig); p += 5; e->size = U48GET(p); p += 6; diff -r 88ea8de5bdf7 sys/src/libventi/fcall.c --- a/sys/src/libventi/fcall.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/fcall.c Tue May 29 00:00:00 2012 +0200 @@ -5,7 +5,7 @@ Packet* vtfcallpack(VtFcall *f) { - uchar buf[4]; + uchar buf[10]; Packet *p; p = packetalloc(); @@ -60,9 +60,17 @@ if(~buf[0] == 0) goto Err; buf[1] = 0; - buf[2] = f->count >> 8; - buf[3] = f->count; - packetappend(p, buf, 4); + if(f->count >= (1<<16)) { + buf[2] = f->count >> 24; + buf[3] = f->count >> 16; + buf[4] = f->count >> 8; + buf[5] = f->count; + packetappend(p, buf, 6); + } else { + buf[2] = f->count >> 8; + buf[3] = f->count; + packetappend(p, buf, 4); + } break; case VtRread: @@ -163,12 +171,25 @@ case VtTread: if(packetconsume(p, f->score, VtScoreSize) < 0 - || packetconsume(p, buf, 4) < 0) + || packetconsume(p, buf, 2) < 0) goto Err; f->blocktype = vtfromdisktype(buf[0]); if(~f->blocktype == 0) goto Err; - f->count = (buf[2] << 8) | buf[3]; + switch(packetsize(p)) { + default: + goto Err; + case 2: + if(packetconsume(p, buf, 2) < 0) + goto Err; + f->count = (buf[0] << 8) | buf[1]; + break; + case 4: + if(packetconsume(p, buf, 4) < 0) + goto Err; + f->count = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + break; + } break; case VtRread: diff -r 88ea8de5bdf7 sys/src/libventi/file.c --- a/sys/src/libventi/file.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/file.c Tue May 29 00:00:00 2012 +0200 @@ -36,7 +36,6 @@ vtfilealloc(VtCache *c, VtBlock *b, VtFile *p, u32int offset, int mode) { int epb; - u32int size; VtEntry e; VtFile *r; @@ -68,21 +67,15 @@ } if(DEPTH(e.type) < sizetodepth(e.size, e.psize, e.dsize)){ - fprint(2, "depth %ud size %llud psize %ud dsize %ud\n", + fprint(2, "depth %ud size %llud psize %lud dsize %lud\n", DEPTH(e.type), e.size, e.psize, e.dsize); werrstr("bad depth"); return nil; } - size = vtcacheblocksize(c); - if(e.dsize > size || e.psize > size){ - werrstr("block sizes %ud, %ud bigger than cache block size %ud", - e.psize, e.dsize, size); - return nil; - } - r = vtmallocz(sizeof(VtFile)); r->c = c; + r->bsize = b->size; r->mode = mode; r->dsize = e.dsize; r->psize = e.psize; @@ -126,7 +119,7 @@ VtBlock *b; VtFile *f; - b = vtcacheallocblock(c, VtDirType); + b = vtcacheallocblock(c, VtDirType, VtEntrySize); if(b == nil) return nil; @@ -191,8 +184,6 @@ u32int offset; assert(ISLOCKED(r)); - assert(psize <= VtMaxLumpSize); - assert(dsize <= VtMaxLumpSize); assert(type == VtDirType || type == VtDataType); if(!r->dir){ @@ -325,12 +316,12 @@ static int shrinksize(VtFile *r, VtEntry *e, uvlong size) { - int i, depth, type, isdir, ppb; + int i, depth, bsiz, type, isdir, ppb; uvlong ptrsz; uchar score[VtScoreSize]; VtBlock *b; - b = vtcacheglobal(r->c, e->score, e->type); + b = vtcacheglobal(r->c, e->score, e->type, r->dsize); if(b == nil) return -1; @@ -342,7 +333,7 @@ ptrsz *= ppb; isdir = r->dir; - while(depth > 0){ + while(DEPTH(type) > 0){ if(b->addr == NilBlock){ /* not worth copying the block just so we can zero some of it */ vtblockput(b); @@ -369,7 +360,11 @@ type--; memmove(score, b->data+i*VtScoreSize, VtScoreSize); vtblockput(b); - b = vtcacheglobal(r->c, score, type); + if(type == VtDataType || type == VtDirType) + bsiz = r->dsize; + else + bsiz = r->psize; + b = vtcacheglobal(r->c, score, type, bsiz); if(b == nil) return -1; } @@ -498,10 +493,10 @@ } static VtBlock * -blockwalk(VtBlock *p, int index, VtCache *c, int mode, VtEntry *e) +blockwalk(VtFile *r, VtBlock *p, int index, VtCache *c, int mode, VtEntry *e) { VtBlock *b; - int type; + int type, size; uchar *score; VtEntry oe; @@ -519,12 +514,16 @@ } /*print("walk from %V/%d ty %d to %V ty %d\n", p->score, index, p->type, score, type); */ + if(type == VtDirType || type == VtDataType) + size = r->dsize; + else + size = r->psize; if(mode == VtOWRITE && vtglobaltolocal(score) == NilBlock){ - b = vtcacheallocblock(c, type); + b = vtcacheallocblock(c, type, size); if(b) goto HaveCopy; }else - b = vtcacheglobal(c, score, type); + b = vtcacheglobal(c, score, type, size); if(b == nil || mode == VtOREAD) return b; @@ -566,7 +565,7 @@ assert(ISLOCKED(r)); assert(depth <= VtPointerDepth); - b = vtcacheglobal(r->c, e->score, e->type); + b = vtcacheglobal(r->c, e->score, e->type, r->dsize); if(b == nil) return -1; @@ -577,7 +576,7 @@ * or an error occurs. */ while(DEPTH(e->type) < depth){ - bb = vtcacheallocblock(r->c, e->type+1); + bb = vtcacheallocblock(r->c, e->type+1, r->psize); if(bb == nil) break; memmove(bb->data, b->score, VtScoreSize); @@ -605,7 +604,7 @@ assert(ISLOCKED(r)); assert(depth <= VtPointerDepth); - rb = vtcacheglobal(r->c, e->score, e->type); + rb = vtcacheglobal(r->c, e->score, e->type, r->psize); if(rb == nil) return -1; @@ -618,7 +617,7 @@ ob = nil; b = rb; for(; DEPTH(e->type) > depth; e->type--){ - nb = vtcacheglobal(r->c, b->data, e->type-1); + nb = vtcacheglobal(r->c, b->data, e->type-1, r->psize); if(nb == nil) break; if(ob!=nil && ob!=rb) @@ -720,7 +719,7 @@ m = VtORDWR; for(i=DEPTH(e.type); i>=0; i--){ - bb = blockwalk(b, index[i], r->c, i==0 ? mode : m, &e); + bb = blockwalk(r, b, index[i], r->c, i==0 ? mode : m, &e); if(bb == nil) goto Err; vtblockput(b); @@ -768,7 +767,7 @@ index[DEPTH(e.type)] = r->offset % r->epb; for(i=DEPTH(e.type); i>=1; i--){ - bb = blockwalk(b, index[i], r->c, VtOREAD, &e); + bb = blockwalk(r, b, index[i], r->c, VtOREAD, &e); if(bb == nil) goto Err; vtblockput(b); @@ -837,7 +836,7 @@ case VtORDWR: assert(r->mode == VtORDWR); if(r->local == 1){ - b = vtcacheglobal(r->c, r->score, VtDirType); + b = vtcacheglobal(r->c, r->score, VtDirType, r->bsize); if(b == nil) return nil; b->pc = getcallerpc(&r); @@ -861,7 +860,7 @@ } addr = vtglobaltolocal(r->score); if(addr == NilBlock) - return vtcacheglobal(r->c, r->score, VtDirType); + return vtcacheglobal(r->c, r->score, VtDirType, r->bsize); b = vtcachelocal(r->c, addr, VtDirType); if(b) @@ -1220,7 +1219,7 @@ */ index[depth] = r->offset % r->epb; for(i=depth; i>=0; i--){ - bb = blockwalk(b, index[i], r->c, VtORDWR, &e); + bb = blockwalk(r, b, index[i], r->c, VtORDWR, &e); if(bb == nil) goto Err; bi[i] = bb; diff -r 88ea8de5bdf7 sys/src/libventi/log.c --- a/sys/src/libventi/log.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/log.c Tue May 29 00:00:00 2012 +0200 @@ -2,8 +2,6 @@ #include #include -char *VtServerLog = "libventi/server"; - int ventilogging; #define log not_the_log_library_call @@ -164,35 +162,16 @@ qunlock(&vl.lk); } -static int -timefmt(Fmt *fmt) -{ - static uvlong t0; - uvlong t; - - if(t0 == 0) - t0 = nsec(); - t = nsec()-t0; - return fmtprint(fmt, "T+%d.%04d", (uint)(t/1000000000), (uint)(t%1000000000)/100000); -} - void vtlogvprint(VtLog *l, char *fmt, va_list arg) { int n; char *p; VtLogChunk *c; - static int first = 1; if(l == nil) return; - - if(first){ - fmtinstall('T', timefmt); - first = 0; - } - - + qlock(&l->lk); c = l->w; n = c->ep - c->wp; diff -r 88ea8de5bdf7 sys/src/libventi/mkfile --- a/sys/src/libventi/mkfile Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/mkfile Tue May 29 00:00:00 2012 +0200 @@ -24,6 +24,7 @@ scorefmt.$O\ send.$O\ server.$O\ + sha1.$O\ srvhello.$O\ strdup.$O\ string.$O\ diff -r 88ea8de5bdf7 sys/src/libventi/root.c --- a/sys/src/libventi/root.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/root.c Tue May 29 00:00:00 2012 +0200 @@ -6,19 +6,30 @@ static int checksize(int n) { - if(n < 256 || n > VtMaxLumpSize) { + if(n < 256) { werrstr("bad block size"); return -1; } return 0; } +extern int vttobig(ulong); + void vtrootpack(VtRoot *r, uchar *p) { uchar *op = p; + int vers, bsize; - U16PUT(p, VtRootVersion); + vers = VtRootVersion; + bsize = r->blocksize; + if(bsize >= (1<<16)) { + vers |= _VtRootVersionBig; + bsize = vttobig(bsize); + if(bsize < 0) + sysfatal("invalid root blocksize: %#lx", r->blocksize); + } + U16PUT(p, vers); p += 2; memmove(p, r->name, sizeof(r->name)); p += sizeof(r->name); @@ -26,7 +37,7 @@ p += sizeof(r->type); memmove(p, r->score, VtScoreSize); p += VtScoreSize; - U16PUT(p, r->blocksize); + U16PUT(p, bsize); p += 2; memmove(p, r->prev, VtScoreSize); p += VtScoreSize; @@ -42,7 +53,7 @@ memset(r, 0, sizeof(*r)); vers = U16GET(p); - if(vers != VtRootVersion) { + if((vers&~_VtRootVersionBig) != VtRootVersion) { werrstr("unknown root version"); return -1; } @@ -56,6 +67,8 @@ memmove(r->score, p, VtScoreSize); p += VtScoreSize; r->blocksize = U16GET(p); + if(vers & _VtRootVersionBig) + r->blocksize = (r->blocksize >> 5) << (r->blocksize & 31); if(checksize(r->blocksize) < 0) return -1; p += 2; diff -r 88ea8de5bdf7 sys/src/libventi/rpc.c --- a/sys/src/libventi/rpc.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/rpc.c Tue May 29 00:00:00 2012 +0200 @@ -37,6 +37,12 @@ uchar tag, buf[2], *top; Rwait *r, *rr; + if(z == nil){ + werrstr("not connected"); + packetfree(p); + return nil; + } + /* must malloc because stack could be private */ r = vtmallocz(sizeof(Rwait)); diff -r 88ea8de5bdf7 sys/src/libventi/send.c --- a/sys/src/libventi/send.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/send.c Tue May 29 00:00:00 2012 +0200 @@ -11,7 +11,7 @@ { IOchunk ioc; int n, tot; - uchar buf[2]; + uchar buf[4]; if(z->state != VtStateConnected) { werrstr("session not connected"); @@ -20,15 +20,24 @@ /* add framing */ n = packetsize(p); - if(n >= (1<<16)) { - werrstr("packet too large"); - packetfree(p); - return -1; + if(z->version[1] == '2') { + if(n >= (1<<16)) { + werrstr("packet too large"); + packetfree(p); + return -1; + } + buf[0] = n>>8; + buf[1] = n; + packetprefix(p, buf, 2); + ventisendbytes += n+2; + } else { + buf[0] = n>>24; + buf[1] = n>>16; + buf[2] = n>>8; + buf[3] = n; + packetprefix(p, buf, 4); + ventisendbytes += n+4; } - buf[0] = n>>8; - buf[1] = n; - packetprefix(p, buf, 2); - ventisendbytes += n+2; ventisendpackets++; tot = 0; @@ -63,7 +72,7 @@ _vtrecv(VtConn *z) { uchar buf[10], *b; - int n; + int n, need; Packet *p; int size, len; @@ -75,11 +84,12 @@ p = z->part; /* get enough for head size */ size = packetsize(p); - while(size < 2) { - b = packettrailer(p, 2); + need = z->version[1] - '0'; // 2 or 4 + while(size < need) { + b = packettrailer(p, need); assert(b != nil); if(0) fprint(2, "%d read hdr\n", getpid()); - n = read(z->infd, b, 2); + n = read(z->infd, b, need); if(0) fprint(2, "%d got %d (%r)\n", getpid(), n); if(n==0 || (n<0 && !interrupted())) goto Err; @@ -87,10 +97,15 @@ packettrim(p, 0, size); } - if(packetconsume(p, buf, 2) < 0) + if(packetconsume(p, buf, need) < 0) goto Err; - len = (buf[0] << 8) | buf[1]; - size -= 2; + if(z->version[1] == '2') { + len = (buf[0] << 8) | buf[1]; + size -= 2; + } else { + len = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + size -= 4; + } while(size < len) { n = len - size; diff -r 88ea8de5bdf7 sys/src/libventi/server.c --- a/sys/src/libventi/server.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/server.c Tue May 29 00:00:00 2012 +0200 @@ -31,6 +31,8 @@ static void listenproc(void*); static void connproc(void*); +char *VtServerLog = "libventi/server"; + static void scincref(VtSconn *sc) { diff -r 88ea8de5bdf7 sys/src/libventi/sha1.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sys/src/libventi/sha1.c Tue May 29 00:00:00 2012 +0200 @@ -0,0 +1,28 @@ +#include +#include +#include +#include + +void +vtsha1(uchar score[VtScoreSize], uchar *p, int n) +{ + DigestState ds; + + memset(&ds, 0, sizeof ds); + sha1(p, n, score, &ds); +} + +int +vtsha1check(uchar score[VtScoreSize], uchar *p, int n) +{ + DigestState ds; + uchar score2[VtScoreSize]; + + memset(&ds, 0, sizeof ds); + sha1(p, n, score2, &ds); + if(memcmp(score, score2, VtScoreSize) != 0) { + werrstr("vtsha1check failed"); + return -1; + } + return 0; +} diff -r 88ea8de5bdf7 sys/src/libventi/time.c --- a/sys/src/libventi/time.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/time.c Tue May 29 00:00:00 2012 +0200 @@ -8,15 +8,18 @@ vlong ns; Tm tm; + if(fmt->flags&FmtSign){ + ns = va_arg(fmt->args, long); + ns *= 1000000000; + } else + ns = nsec(); + tm = *localtime(ns/1000000000); if(fmt->flags&FmtLong){ - ns = nsec(); - tm = *localtime(ns/1000000000); return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d.%03d", tm.year+1900, tm.mon+1, tm.mday, tm.hour, tm.min, tm.sec, (int)(ns%1000000000)/1000000); }else{ - tm = *localtime(time(0)); return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d", tm.year+1900, tm.mon+1, tm.mday, tm.hour, tm.min, tm.sec); diff -r 88ea8de5bdf7 sys/src/libventi/version.c --- a/sys/src/libventi/version.c Sat May 26 00:00:00 2012 +0200 +++ b/sys/src/libventi/version.c Tue May 29 00:00:00 2012 +0200 @@ -3,6 +3,7 @@ #include static char *okvers[] = { + "04", "02", nil, };