Tcl Source Code

Check-in [f85e0e4fe4]
Login
Bounty program for improvements to Tcl and certain Tcl packages.
Tcl 2019 Conference, Houston/TX, US, Nov 4-8
Send your abstracts to [email protected]
or submit via the online form by Sep 9.

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:[b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f85e0e4fe4f89cf8f24f7b3ec9c7b7b0ce131383
User & Date: dkf 2016-10-30 05:05:46
Context
2016-10-30
08:31
[1ae12987cb] Ensure that deleting the [history] command deletes its storage. check-in: eb11bf9cb5 user: dkf tags: trunk
05:05
[b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: f85e0e4fe4 user: dkf tags: trunk
05:04
[b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: 83a6a4d431 user: dkf tags: core-8-6-branch
2016-10-28
15:46
Add optimization to [dict append]. check-in: df0c2fc07c user: dgp tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to generic/tclZlib.c.

   173    173   static Tcl_ObjCmdProc		ZlibStreamAddCmd;
   174    174   static Tcl_ObjCmdProc		ZlibStreamHeaderCmd;
   175    175   static Tcl_ObjCmdProc		ZlibStreamPutCmd;
   176    176   
   177    177   static void		ConvertError(Tcl_Interp *interp, int code,
   178    178   			    uLong adler);
   179    179   static Tcl_Obj *	ConvertErrorToList(int code, uLong adler);
          180  +static inline int	Deflate(z_streamp strm, void *bufferPtr,
          181  +			    int bufferSize, int flush, int *writtenPtr);
   180    182   static void		ExtractHeader(gz_header *headerPtr, Tcl_Obj *dictObj);
   181    183   static int		GenerateHeader(Tcl_Interp *interp, Tcl_Obj *dictObj,
   182    184   			    GzipHeader *headerPtr, int *extraSizePtr);
   183    185   static int		ZlibPushSubcmd(Tcl_Interp *interp, int objc,
   184    186   			    Tcl_Obj *const objv[]);
   185    187   static inline int	ResultCopy(ZlibChannelData *cd, char *buf,
   186    188   			    int toRead);
................................................................................
   574    576       }
   575    577   
   576    578       if (latin1enc != NULL) {
   577    579   	Tcl_FreeEncoding(latin1enc);
   578    580       }
   579    581   }
   580    582   
          583  +/*
          584  + * Disentangle the worst of how the zlib API is used.
          585  + */
          586  +
   581    587   static int
   582    588   SetInflateDictionary(
   583    589       z_streamp strm,
   584    590       Tcl_Obj *compDictObj)
   585    591   {
   586    592       if (compDictObj != NULL) {
   587    593   	int length;
................................................................................
   601    607   	int length;
   602    608   	unsigned char *bytes = Tcl_GetByteArrayFromObj(compDictObj, &length);
   603    609   
   604    610   	return deflateSetDictionary(strm, bytes, (unsigned) length);
   605    611       }
   606    612       return Z_OK;
   607    613   }
          614  +
          615  +static inline int
          616  +Deflate(
          617  +    z_streamp strm,
          618  +    void *bufferPtr,
          619  +    int bufferSize,
          620  +    int flush,
          621  +    int *writtenPtr)
          622  +{
          623  +    int e;
          624  +
          625  +    strm->next_out = (Bytef *) bufferPtr;
          626  +    strm->avail_out = (unsigned) bufferSize;
          627  +    e = deflate(strm, flush);
          628  +    if (writtenPtr != NULL) {
          629  +	*writtenPtr = bufferSize - strm->avail_out;
          630  +    }
          631  +    return e;
          632  +}
          633  +
          634  +static inline void
          635  +AppendByteArray(
          636  +    Tcl_Obj *listObj,
          637  +    void *buffer,
          638  +    int size)
          639  +{
          640  +    if (size > 0) {
          641  +	Tcl_Obj *baObj = Tcl_NewByteArrayObj((unsigned char *) buffer, size);
          642  +
          643  +	Tcl_ListObjAppendElement(NULL, listObj, baObj);
          644  +    }
          645  +}
   608    646   
   609    647   /*
   610    648    *----------------------------------------------------------------------
   611    649    *
   612    650    * Tcl_ZlibStreamInit --
   613    651    *
   614    652    *	This command initializes a (de)compression context/handle for
................................................................................
  1135   1173    *
  1136   1174    *	Add data to the stream for compression or decompression from a
  1137   1175    *	bytearray Tcl_Obj.
  1138   1176    *
  1139   1177    *----------------------------------------------------------------------
  1140   1178    */
  1141   1179   
         1180  +#define BUFFER_SIZE_LIMIT	0xFFFF
         1181  +
  1142   1182   int
  1143   1183   Tcl_ZlibStreamPut(
  1144   1184       Tcl_ZlibStream zshandle,	/* As obtained from Tcl_ZlibStreamInit */
  1145   1185       Tcl_Obj *data,		/* Data to compress/decompress */
  1146   1186       int flush)			/* TCL_ZLIB_NO_FLUSH, TCL_ZLIB_FLUSH,
  1147   1187   				 * TCL_ZLIB_FULLFLUSH, or TCL_ZLIB_FINALIZE */
  1148   1188   {
  1149   1189       ZlibStreamHandle *zshPtr = (ZlibStreamHandle *) zshandle;
  1150   1190       char *dataTmp = NULL;
  1151         -    int e, size, outSize;
  1152         -    Tcl_Obj *obj;
         1191  +    int e, size, outSize, toStore;
  1153   1192   
  1154   1193       if (zshPtr->streamEnd) {
  1155   1194   	if (zshPtr->interp) {
  1156   1195   	    Tcl_SetObjResult(zshPtr->interp, Tcl_NewStringObj(
  1157   1196   		    "already past compressed stream end", -1));
  1158   1197   	    Tcl_SetErrorCode(zshPtr->interp, "TCL", "ZIP", "CLOSED", NULL);
  1159   1198   	}
................................................................................
  1171   1210   	if (size == 0) {
  1172   1211   	    return TCL_OK;
  1173   1212   	}
  1174   1213   
  1175   1214   	if (HaveDictToSet(zshPtr)) {
  1176   1215   	    e = SetDeflateDictionary(&zshPtr->stream, zshPtr->compDictObj);
  1177   1216   	    if (e != Z_OK) {
  1178         -		if (zshPtr->interp) {
  1179         -		    ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
  1180         -		}
         1217  +		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
  1181   1218   		return TCL_ERROR;
  1182   1219   	    }
  1183   1220   	    DictWasSet(zshPtr);
  1184   1221   	}
  1185   1222   
  1186   1223   	/*
  1187         -	 * Deflatebound doesn't seem to take various header sizes into
  1188         -	 * account, so we add 100 extra bytes.
         1224  +	 * deflateBound() doesn't seem to take various header sizes into
         1225  +	 * account, so we add 100 extra bytes. However, we can also loop
         1226  +	 * around again so we also set an upper bound on the output buffer
         1227  +	 * size.
  1189   1228   	 */
  1190   1229   
  1191         -	outSize = deflateBound(&zshPtr->stream, zshPtr->stream.avail_in)+100;
  1192         -	zshPtr->stream.avail_out = outSize;
  1193         -	dataTmp = ckalloc(zshPtr->stream.avail_out);
  1194         -	zshPtr->stream.next_out = (Bytef *) dataTmp;
         1230  +	outSize = deflateBound(&zshPtr->stream, size) + 100;
         1231  +	if (outSize > BUFFER_SIZE_LIMIT) {
         1232  +	    outSize = BUFFER_SIZE_LIMIT;
         1233  +	}
         1234  +	dataTmp = ckalloc(outSize);
  1195   1235   
  1196         -	e = deflate(&zshPtr->stream, flush);
  1197         -	while (e == Z_BUF_ERROR || (flush == Z_FINISH && e == Z_OK)) {
         1236  +	while (1) {
         1237  +	    e = Deflate(&zshPtr->stream, dataTmp, outSize, flush, &toStore);
         1238  +
         1239  +	    /*
         1240  +	     * Test if we've filled the buffer up and have to ask deflate() to
         1241  +	     * give us some more. Note that the condition for needing to
         1242  +	     * repeat a buffer transfer when the result is Z_OK is whether
         1243  +	     * there is no more space in the buffer we provided; the zlib
         1244  +	     * library does not necessarily return a different code in that
         1245  +	     * case. [Bug b26e38a3e4] [Tk Bug 10f2e7872b]
         1246  +	     */
         1247  +
         1248  +	    if ((e != Z_BUF_ERROR) && (e != Z_OK || toStore < outSize)) {
         1249  +		if ((e == Z_OK) || (flush == Z_FINISH && e == Z_STREAM_END)) {
         1250  +		    break;
         1251  +		}
         1252  +		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
         1253  +		return TCL_ERROR;
         1254  +	    }
         1255  +
  1198   1256   	    /*
  1199   1257   	     * Output buffer too small to hold the data being generated or we
  1200   1258   	     * are doing the end-of-stream flush (which can spit out masses of
  1201   1259   	     * data). This means we need to put a new buffer into place after
  1202   1260   	     * saving the old generated data to the outData list.
  1203   1261   	     */
  1204   1262   
  1205         -	    obj = Tcl_NewByteArrayObj((unsigned char *) dataTmp, outSize);
  1206         -	    Tcl_ListObjAppendElement(NULL, zshPtr->outData, obj);
         1263  +	    AppendByteArray(zshPtr->outData, dataTmp, outSize);
  1207   1264   
  1208         -	    if (outSize < 0xFFFF) {
  1209         -		outSize = 0xFFFF;	/* There may be *lots* of data left to
  1210         -					 * output... */
         1265  +	    if (outSize < BUFFER_SIZE_LIMIT) {
         1266  +		outSize = BUFFER_SIZE_LIMIT;
         1267  +		/* There may be *lots* of data left to output... */
  1211   1268   		dataTmp = ckrealloc(dataTmp, outSize);
  1212   1269   	    }
  1213         -	    zshPtr->stream.avail_out = outSize;
  1214         -	    zshPtr->stream.next_out = (Bytef *) dataTmp;
  1215         -
  1216         -	    e = deflate(&zshPtr->stream, flush);
  1217         -	}
  1218         -
  1219         -	if (e != Z_OK && !(flush==Z_FINISH && e==Z_STREAM_END)) {
  1220         -	    if (zshPtr->interp) {
  1221         -		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
  1222         -	    }
  1223         -	    return TCL_ERROR;
  1224   1270   	}
  1225   1271   
  1226   1272   	/*
  1227         -	 * And append the final data block.
         1273  +	 * And append the final data block to the outData list.
  1228   1274   	 */
  1229   1275   
  1230         -	if (outSize - zshPtr->stream.avail_out > 0) {
  1231         -	    obj = Tcl_NewByteArrayObj((unsigned char *) dataTmp,
  1232         -		    outSize - zshPtr->stream.avail_out);
  1233         -
  1234         -	    /*
  1235         -	     * Now append the compressed data to the outData list.
  1236         -	     */
  1237         -
  1238         -	    Tcl_ListObjAppendElement(NULL, zshPtr->outData, obj);
  1239         -	}
  1240         -
  1241         -	if (dataTmp) {
  1242         -	    ckfree(dataTmp);
  1243         -	}
         1276  +	AppendByteArray(zshPtr->outData, dataTmp, toStore);
         1277  +	ckfree(dataTmp);
  1244   1278       } else {
  1245   1279   	/*
  1246   1280   	 * This is easy. Just append to the inData list.
  1247   1281   	 */
  1248   1282   
  1249   1283   	Tcl_ListObjAppendElement(NULL, zshPtr->inData, data);
  1250   1284   
................................................................................
  1352   1386   	 * (You can't do it in response to getting Z_NEED_DATA as raw streams
  1353   1387   	 * don't ever issue that.)
  1354   1388   	 */
  1355   1389   
  1356   1390   	if (IsRawStream(zshPtr) && HaveDictToSet(zshPtr)) {
  1357   1391   	    e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj);
  1358   1392   	    if (e != Z_OK) {
  1359         -		if (zshPtr->interp) {
  1360         -		    ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
  1361         -		}
         1393  +		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
  1362   1394   		return TCL_ERROR;
  1363   1395   	    }
  1364   1396   	    DictWasSet(zshPtr);
  1365   1397   	}
  1366   1398   	e = inflate(&zshPtr->stream, zshPtr->flush);
  1367   1399   	if (e == Z_NEED_DICT && HaveDictToSet(zshPtr)) {
  1368   1400   	    e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj);
................................................................................
  2875   2907       /*
  2876   2908        * Flush any data waiting to be compressed.
  2877   2909        */
  2878   2910   
  2879   2911       if (cd->mode == TCL_ZLIB_STREAM_DEFLATE) {
  2880   2912   	cd->outStream.avail_in = 0;
  2881   2913   	do {
  2882         -	    cd->outStream.next_out = (Bytef *) cd->outBuffer;
  2883         -	    cd->outStream.avail_out = (unsigned) cd->outAllocated;
  2884         -	    e = deflate(&cd->outStream, Z_FINISH);
  2885         -	    written = cd->outAllocated - cd->outStream.avail_out;
         2914  +	    e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated,
         2915  +		    Z_FINISH, &written);
  2886   2916   
  2887   2917   	    /*
  2888   2918   	     * Can't be sure that deflate() won't declare the buffer to be
  2889   2919   	     * full (with Z_BUF_ERROR) so handle that case.
  2890   2920   	     */
  2891   2921   
  2892   2922   	    if (e == Z_BUF_ERROR) {
................................................................................
  3082   3112   	return outProc(Tcl_GetChannelInstanceData(cd->parent), buf, toWrite,
  3083   3113   		errorCodePtr);
  3084   3114       }
  3085   3115   
  3086   3116       cd->outStream.next_in = (Bytef *) buf;
  3087   3117       cd->outStream.avail_in = toWrite;
  3088   3118       do {
  3089         -	cd->outStream.next_out = (Bytef *) cd->outBuffer;
  3090         -	cd->outStream.avail_out = cd->outAllocated;
  3091         -
  3092         -	e = deflate(&cd->outStream, Z_NO_FLUSH);
  3093         -	produced = cd->outAllocated - cd->outStream.avail_out;
         3119  +	e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated,
         3120  +		Z_NO_FLUSH, &produced);
  3094   3121   
  3095   3122   	if ((e == Z_OK && produced > 0) || e == Z_BUF_ERROR) {
  3096   3123   	    /*
  3097   3124   	     * deflate() indicates that it is out of space by returning
  3098         -	     * Z_BUF_ERROR; in that case, we must write the whole buffer out
  3099         -	     * and retry to compress what is left.
         3125  +	     * Z_BUF_ERROR *or* by simply returning Z_OK with no remaining
         3126  +	     * space; in either case, we must write the whole buffer out and
         3127  +	     * retry to compress what is left.
  3100   3128   	     */
  3101   3129   
  3102   3130   	    if (e == Z_BUF_ERROR) {
  3103   3131   		produced = cd->outAllocated;
  3104   3132   		e = Z_OK;
  3105   3133   	    }
  3106   3134   	    if (Tcl_WriteRaw(cd->parent, cd->outBuffer, produced) < 0) {

Changes to tests/zlib.test.

   134    134   } {{} 69f34b6a abcdeEDCBA..}
   135    135   test zlib-7.7 {zlib stream: Bug 25842c161} -constraints zlib -body {
   136    136       set s [zlib stream deflate]
   137    137       $s put {}
   138    138   } -cleanup {
   139    139       catch {$s close}
   140    140   } -result ""
          141  +# Also causes Tk Bug 10f2e7872b
          142  +test zlib-7.8 {zlib stream: Bug b26e38a3e4} -constraints zlib -setup {
          143  +    expr srand(12345)
          144  +    set randdata {}
          145  +    for {set i 0} {$i<6001} {incr i} {
          146  +	append randdata [binary format c [expr {int(256*rand())}]]
          147  +    }
          148  +} -body {
          149  +    set strm [zlib stream compress]
          150  +    for {set i 1} {$i<3000} {incr i} {
          151  +	$strm put $randdata
          152  +    }
          153  +    $strm put -finalize $randdata
          154  +    set data [$strm get]
          155  +    list [string length $data] [string length [zlib decompress $data]]
          156  +} -cleanup {
          157  +    catch {$strm close}
          158  +    unset -nocomplain randdata data
          159  +} -result {120185 18003000}
   141    160   
   142    161   test zlib-8.1 {zlib transformation} -constraints zlib -setup {
   143    162       set file [makeFile {} test.gz]
   144    163   } -body {
   145    164       set f [zlib push gzip [open $file w] -header {comment gorp}]
   146    165       puts $f "ok"
   147    166       close $f