Further improvements to the fuzzer. It still is not quite working. Pausing

to work on other things....

FossilOrigin-Name: 5f2f2fce40f43debeb0492c9b460b85c7dad2bde

manifest

@@ -1,8 +1,5 @@
------BEGIN PGP SIGNED MESSAGE-----
-Hash: SHA1
-
-C Added\smost\sof\sthe\slogic.\s\sSimple\stest\sruns\swithout\ssegfaulting\sbut\sdoes\snot\ngive\sthe\scorrect\sanswer.
-D 2011-03-26T19:04:47.346
+C Further\simprovements\sto\sthe\sfuzzer.\s\sIt\sstill\sis\snot\squite\sworking.\s\sPausing\nto\swork\son\sother\sthings....
+D 2011-03-29T14:08:09.188
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 6c96e694f446500449f683070b906de9fce17b88
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -102,7 +99,7 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea
 F ext/rtree/sqlite3rtree.h 1af0899c63a688e272d69d8e746f24e76f10a3f0
 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
-F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
+F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F main.mk a767e12162f02719fa94697a6ff0c8b51bcd62a6
 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
@@ -205,7 +202,7 @@ F src/test_config.c 62f0f8f934b1d5c7e4cd4f506ae453a1117b47d7
 F src/test_demovfs.c 0aed671636735116fc872c5b03706fd5612488b5
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_func.c cbdec5cededa0761daedde5baf06004a9bf416b5
-F src/test_fuzzer.c 133c830fdd4342b687910a04cf8e617e0f5f0e5f
+F src/test_fuzzer.c 3e402dd5e6c4f096fdc92c52e775e8fad85bce9b
 F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2
 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c
 F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99
@@ -479,7 +476,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167
 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5
 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b
 F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c
-F test/fuzzer1.test e0fe96bb8d318250b35407954c7059eea8ea77b2
+F test/fuzzer1.test a5d60f618443b86b5f5a695a41d01b7d8697345d
 F test/hook.test f04c3412463f8ec117c1c704c74ca0f627ce733a
 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4
 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d
@@ -608,7 +605,7 @@ F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6
 F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850
 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
-F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 x
+F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
 F test/quick.test 1681febc928d686362d50057c642f77a02c62e57
 F test/quota.test ddafe133653093eb9a99ccd6264884ae43f9c9b8
@@ -896,7 +893,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/lemon.c dfd81a51b6e27e469ba21d01a75ddf092d429027
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
 F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309
-F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x
+F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c.tcl cf44512a48112b1ba09590548660a5a6877afdb3
 F tool/mksqlite3h.tcl d76c226a5e8e1f3b5f6593bcabe5e98b3b1ec9ff
@@ -921,14 +918,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P ea3a4ee136ff6699c3099178f0efaa8bb517715f
-R 6a09f2dbc53bf6a269c56104914ae3e5
+P fb4c31eac8a7290f61c50a3552245660e1271871
+R 86b6649d084e41c015babc3a0e3fb294
 U drh
-Z c40c33ca74d9ed106c795140b6ad71ee
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFNjjjSoxKgR168RlERAjfMAKCJigUMIbnNV83nhVCWCpDK5WxPFQCeLIbS
-+J80DLXre8S3k4SR8glB8Jc=
-=nBDg
------END PGP SIGNATURE-----
+Z 67d6460bdc70fd366585e4c8607ab8f4

manifest.uuid

@@ -1 +1 @@
-fb4c31eac8a7290f61c50a3552245660e1271871
+5f2f2fce40f43debeb0492c9b460b85c7dad2bde

src/test_fuzzer.c

@@ -17,6 +17,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include <stdio.h>
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
@@ -49,7 +50,15 @@ struct fuzzer_rule {
 };
 
 /*
-** A stem object is used to generate variants.  
+** A stem object is used to generate variants.  It is also used to record
+** previously generated outputs.
+**
+** Every stem is added to a hash table as it is output.  Generation of
+** duplicate stems is suppressed.
+**
+** Active stems (those that might generate new outputs) are kepts on a linked
+** list sorted by increasing cost.  The cost is the sum of rBaseCost and
+** pRule->rCost.
 */
 struct fuzzer_stem {
   char *zBasis;              /* Word being fuzzed */
@@ -83,6 +92,7 @@ struct fuzzer_cursor {
   fuzzer_stem *pDone;        /* Stems already processed to completion */
   char *zBuf;                /* Temporary use buffer */
   int nBuf;                  /* Bytes allocated for zBuf */
+  fuzzer_rule nullRule;      /* Null rule used first */
   fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */
 };
 
@@ -171,7 +181,6 @@ static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
   memset(pCur, 0, sizeof(*pCur));
   pCur->pVtab = p;
   *ppCursor = &pCur->base;
-  p->nCursor++;
   if( p->nCursor==0 && p->pNewRule ){
     unsigned int i;
     fuzzer_rule *pX;
@@ -191,6 +200,7 @@ static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
     }
     p->pRule = fuzzerMergeRules(p->pRule, pX);
   }
+  p->nCursor++;
   return SQLITE_OK;
 }
 
@@ -221,35 +231,34 @@ static int fuzzerClose(sqlite3_vtab_cursor *cur){
   fuzzerClearCursor(pCur, 0);
   sqlite3_free(pCur->zBuf);
   pCur->pVtab->nCursor--;
+  sqlite3_free(pCur);
   return SQLITE_OK;
 }
 
 /*
 ** Compute the current output term for a fuzzer_stem.
 */
-static int fuzzerComputeWord(
-  fuzzer_cursor *pCur,
-  fuzzer_stem *pStem
+static int fuzzerRender(
+  fuzzer_stem *pStem,   /* The stem to be rendered */
+  char **pzBuf,         /* Write results into this buffer.  realloc if needed */
+  int *pnBuf            /* Size of the buffer */
 ){
   const fuzzer_rule *pRule = pStem->pRule;
   int n;
+  char *z;
 
-  n = pStem->nBasis;
-  if( pStem->n>=0 ) n += pRule->nTo - pRule->nFrom;
-  if( pCur->nBuf<n+1 ){
-    pCur->zBuf = sqlite3_realloc(pCur->zBuf, n+100);
-    if( pCur->zBuf==0 ) return SQLITE_NOMEM;
-    pCur->nBuf = n+100;
+  n = pStem->nBasis + pRule->nTo - pRule->nFrom;
+  if( (*pnBuf)<n+1 ){
+    (*pzBuf) = sqlite3_realloc((*pzBuf), n+100);
+    if( (*pzBuf)==0 ) return SQLITE_NOMEM;
+    (*pnBuf) = n+100;
   }
   n = pStem->n;
-  if( n<0 ){
-    memcpy(pCur->zBuf, pStem->zBasis, pStem->nBasis+1);
-  }else{
-    memcpy(pCur->zBuf, pStem->zBasis, n);
-    memcpy(&pCur->zBuf[n], pRule->zTo, pRule->nTo);
-    memcpy(&pCur->zBuf[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], 
-           pStem->nBasis-n-pRule->nFrom+1);
-  }
+  z = *pzBuf;
+  memcpy(z, pStem->zBasis, n);
+  memcpy(&z[n], pRule->zTo, pRule->nTo);
+  memcpy(&z[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], 
+         pStem->nBasis-n-pRule->nFrom+1);
   return SQLITE_OK;
 }
 
@@ -260,7 +269,7 @@ static int fuzzerComputeWord(
 static unsigned int fuzzerHash(const char *z){
   unsigned int h = 0;
   while( *z ){ h = (h<<3) ^ (h>>29) ^ *(z++); }
-  return h%10007;
+  return h % FUZZER_HASH;
 }
 
 /*
@@ -270,9 +279,30 @@ static fuzzer_cost fuzzerCost(fuzzer_stem *pStem){
   return pStem->rBaseCost + pStem->pRule->rCost;
 }
 
+/*
+** Return 1 if the string to which the cursor is point has already
+** been emitted.  Return 0 if not.  Return -1 on a memory allocation
+** failures.
+*/
+static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){
+  unsigned int h;
+  fuzzer_stem *pLookup;
+
+  if( fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){
+    return -1;
+  }
+  h = fuzzerHash(pCur->zBuf);
+  pLookup = pCur->apHash[h];
+    while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){
+    pLookup = pLookup->pHash;
+  }
+  return pLookup!=0;
+}
+
 /*
 ** Advance a fuzzer_stem to its next value.   Return 0 if there are
-** no more values that can be generated by this fuzzer_stem.
+** no more values that can be generated by this fuzzer_stem.  Return
+** -1 on a memory allocation failure.
 */
 static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){
   const fuzzer_rule *pRule;
@@ -283,21 +313,14 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){
        || memcmp(&pStem->zBasis[pStem->n], pRule->zFrom, pRule->nFrom)==0
       ){
         /* Found a rewrite case.  Make sure it is not a duplicate */
-        unsigned int h;
-        fuzzer_stem *pLookup;
-
-        fuzzerComputeWord(pCur, pStem);
-        h = fuzzerHash(pCur->zBuf);
-        pLookup = pCur->apHash[h];
-        while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){
-          pLookup = pLookup->pHash;
-        }
-        if( pLookup==0 ) return 1;  /* A new output is found. */
+        int rc = fuzzerSeen(pCur, pStem);
+        if( rc<0 ) return -1;
+        if( rc==0 ) return 1;  
       }
     }
     pStem->n = -1;
     pStem->pRule = pRule->pNext;
-    if( fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0;
+    if( pStem->pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0;
   }
   return 0;
 }
@@ -310,6 +333,10 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){
 static fuzzer_stem *fuzzerInsert(fuzzer_stem *pList, fuzzer_stem *pNew){
   fuzzer_cost c1;
 
+  if( pList==0 ){
+    pNew->pNext = 0;
+    return pNew;
+  }
   c1 = fuzzerCost(pNew);
   if( c1 <= fuzzerCost(pList) ){
     pNew->pNext = pList;
@@ -358,20 +385,30 @@ static fuzzer_stem *fuzzerNewStem(
 ** Advance a cursor to its next row of output
 */
 static int fuzzerNext(sqlite3_vtab_cursor *cur){
-  fuzzer_cursor *pCur = (fuzzer_cursor*)pCur;
+  fuzzer_cursor *pCur = (fuzzer_cursor*)cur;
+  int rc;
   fuzzer_stem *pStem, *pNew;
 
   /* Use the element the cursor is currently point to to create
   ** a new stem and insert the new stem into the priority queue.
   */
-  fuzzerComputeWord(pCur, pCur->pStem);
-  pNew = fuzzerNewStem(pCur, pCur->zBuf, fuzzerCost(pCur->pStem));
-  if( pNew ){
-    if( fuzzerAdvance(pCur, pNew)==0 ){
-      pNew->pNext = pCur->pDone;
-      pCur->pDone = pNew;
+  pStem = pCur->pStem;
+  if( fuzzerCost(pStem)>0 ){
+    rc = fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf);
+    if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM;
+    pNew = fuzzerNewStem(pCur, pCur->zBuf, fuzzerCost(pStem));
+    if( pNew ){
+      if( fuzzerAdvance(pCur, pNew)==0 ){
+        pNew->pNext = pCur->pDone;
+        pCur->pDone = pNew;
+      }else{
+        pCur->pStem = fuzzerInsert(pStem, pNew);
+        if( pCur->pStem==pNew ){
+          return SQLITE_OK;
+        }
+      }
     }else{
-      pCur->pStem = fuzzerInsert(pCur->pStem, pNew);
+      return SQLITE_NOMEM;
     }
   }
 
@@ -381,7 +418,12 @@ static int fuzzerNext(sqlite3_vtab_cursor *cur){
   while( (pStem = pCur->pStem)!=0 ){
     if( fuzzerAdvance(pCur, pStem) ){
       pCur->pStem = fuzzerInsert(pStem->pNext, pStem);
-      return SQLITE_OK;  /* New word found */
+      if( pCur->pStem!=pStem && (rc = fuzzerSeen(pCur, pStem))!=0 ){
+        if( rc<0 ) return SQLITE_NOMEM;
+        continue;
+      }else{
+        return SQLITE_OK;  /* New word found */
+      }
     }
     pCur->pStem = pStem->pNext;
     pStem->pNext = pCur->pDone;
@@ -406,9 +448,10 @@ static int fuzzerFilter(
 ){
   fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor;
   const char *zWord = 0;
-  pCur->rLimit = 2147483647;
+  fuzzer_stem *pStem;
 
   fuzzerClearCursor(pCur, 1);
+  pCur->rLimit = 2147483647;
   if( idxNum==1 ){
     zWord = (const char*)sqlite3_value_text(argv[0]);
   }else if( idxNum==2 ){
@@ -418,8 +461,15 @@ static int fuzzerFilter(
     pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[1]);
   }
   if( zWord==0 ) zWord = "";
-  pCur->pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0);
-  if( pCur->pStem==0 ) return SQLITE_NOMEM;
+  pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0);
+  if( pStem==0 ) return SQLITE_NOMEM;
+  pCur->nullRule.pNext = pCur->pVtab->pRule;
+  pCur->nullRule.rCost = 0;
+  pCur->nullRule.nFrom = 0;
+  pCur->nullRule.nTo = 0;
+  pCur->nullRule.zFrom = "";
+  pStem->pRule = &pCur->nullRule;
+  pStem->n = pStem->nBasis;
   return SQLITE_OK;
 }
 
@@ -431,7 +481,7 @@ static int fuzzerColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
   fuzzer_cursor *pCur = (fuzzer_cursor*)cur;
   if( i==0 ){
     /* the "word" column */
-    if( fuzzerComputeWord(pCur, pCur->pStem)==SQLITE_NOMEM ){
+    if( fuzzerRender(pCur->pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){
       return SQLITE_NOMEM;
     }
     sqlite3_result_text(ctx, pCur->zBuf, -1, SQLITE_TRANSIENT);
@@ -567,17 +617,16 @@ static int fuzzerUpdate(
     pVTab->zErrMsg = sqlite3_mprintf("cost must be positive");
     return SQLITE_CONSTRAINT;    
   }
-  nFrom = strlen(zFrom)+1;
-  nTo = strlen(zTo)+1;
-  if( nTo<4 ) nTo = 4;
-  pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo - 4 );
+  nFrom = strlen(zFrom);
+  nTo = strlen(zTo);
+  pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo );
   if( pRule==0 ){
     return SQLITE_NOMEM;
   }
-  pRule->zFrom = &pRule->zTo[nTo];
+  pRule->zFrom = &pRule->zTo[nTo+1];
   pRule->nFrom = nFrom;
-  memcpy(pRule->zFrom, zFrom, nFrom);
-  memcpy(pRule->zTo, zTo, nTo);
+  memcpy(pRule->zFrom, zFrom, nFrom+1);
+  memcpy(pRule->zTo, zTo, nTo+1);
   pRule->nTo = nTo;
   pRule->rCost = rCost;
   pRule->pNext = p->pNewRule;

test/fuzzer1.test

@@ -32,8 +32,8 @@ do_test fuzzer1-1.1 {
 do_test fuzzer1-1.2 {
   db eval {
     INSERT INTO f1(cfrom, cto, cost) VALUES('e','a',1);
-    INSERT INTO f1(cfrom, cto, cost) VALUES('a','e',1);
-    INSERT INTO f1(cfrom, cto, cost) VALUES('e','o',2);
+    INSERT INTO f1(cfrom, cto, cost) VALUES('a','e',10);
+    INSERT INTO f1(cfrom, cto, cost) VALUES('e','o',100);
   }
 } {}