Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | [b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
f85e0e4fe4f89cf8f24f7b3ec9c7b7b0 |
User & Date: | dkf 2016-10-30 05:05:46.089 |
Context
2016-10-30
| ||
08:31 | [1ae12987cb] Ensure that deleting the [history] command deletes its storage. check-in: eb11bf9cb5 user: dkf tags: trunk | |
05:05 | [b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: f85e0e4fe4 user: dkf tags: trunk | |
05:04 | [b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: 83a6a4d431 user: dkf tags: core-8-6-branch | |
2016-10-28
| ||
15:46 | Add optimization to [dict append]. check-in: df0c2fc07c user: dgp tags: trunk | |
Changes
Changes to generic/tclZlib.c.
︙ | ︙ | |||
173 174 175 176 177 178 179 180 181 182 183 184 185 186 | static Tcl_ObjCmdProc ZlibStreamAddCmd; static Tcl_ObjCmdProc ZlibStreamHeaderCmd; static Tcl_ObjCmdProc ZlibStreamPutCmd; static void ConvertError(Tcl_Interp *interp, int code, uLong adler); static Tcl_Obj * ConvertErrorToList(int code, uLong adler); static void ExtractHeader(gz_header *headerPtr, Tcl_Obj *dictObj); static int GenerateHeader(Tcl_Interp *interp, Tcl_Obj *dictObj, GzipHeader *headerPtr, int *extraSizePtr); static int ZlibPushSubcmd(Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); static inline int ResultCopy(ZlibChannelData *cd, char *buf, int toRead); | > > | 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | static Tcl_ObjCmdProc ZlibStreamAddCmd; static Tcl_ObjCmdProc ZlibStreamHeaderCmd; static Tcl_ObjCmdProc ZlibStreamPutCmd; static void ConvertError(Tcl_Interp *interp, int code, uLong adler); static Tcl_Obj * ConvertErrorToList(int code, uLong adler); static inline int Deflate(z_streamp strm, void *bufferPtr, int bufferSize, int flush, int *writtenPtr); static void ExtractHeader(gz_header *headerPtr, Tcl_Obj *dictObj); static int GenerateHeader(Tcl_Interp *interp, Tcl_Obj *dictObj, GzipHeader *headerPtr, int *extraSizePtr); static int ZlibPushSubcmd(Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); static inline int ResultCopy(ZlibChannelData *cd, char *buf, int toRead); |
︙ | ︙ | |||
574 575 576 577 578 579 580 581 582 583 584 585 586 587 | } if (latin1enc != NULL) { Tcl_FreeEncoding(latin1enc); } } static int SetInflateDictionary( z_streamp strm, Tcl_Obj *compDictObj) { if (compDictObj != NULL) { int length; | > > > > | 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 | } if (latin1enc != NULL) { Tcl_FreeEncoding(latin1enc); } } /* * Disentangle the worst of how the zlib API is used. */ static int SetInflateDictionary( z_streamp strm, Tcl_Obj *compDictObj) { if (compDictObj != NULL) { int length; |
︙ | ︙ | |||
601 602 603 604 605 606 607 608 609 610 611 612 613 614 | int length; unsigned char *bytes = Tcl_GetByteArrayFromObj(compDictObj, &length); return deflateSetDictionary(strm, bytes, (unsigned) length); } return Z_OK; } /* *---------------------------------------------------------------------- * * Tcl_ZlibStreamInit -- * * This command initializes a (de)compression context/handle for | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 | int length; unsigned char *bytes = Tcl_GetByteArrayFromObj(compDictObj, &length); return deflateSetDictionary(strm, bytes, (unsigned) length); } return Z_OK; } static inline int Deflate( z_streamp strm, void *bufferPtr, int bufferSize, int flush, int *writtenPtr) { int e; strm->next_out = (Bytef *) bufferPtr; strm->avail_out = (unsigned) bufferSize; e = deflate(strm, flush); if (writtenPtr != NULL) { *writtenPtr = bufferSize - strm->avail_out; } return e; } static inline void AppendByteArray( Tcl_Obj *listObj, void *buffer, int size) { if (size > 0) { Tcl_Obj *baObj = Tcl_NewByteArrayObj((unsigned char *) buffer, size); Tcl_ListObjAppendElement(NULL, listObj, baObj); } } /* *---------------------------------------------------------------------- * * Tcl_ZlibStreamInit -- * * This command initializes a (de)compression context/handle for |
︙ | ︙ | |||
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 | * * Add data to the stream for compression or decompression from a * bytearray Tcl_Obj. * *---------------------------------------------------------------------- */ int Tcl_ZlibStreamPut( Tcl_ZlibStream zshandle, /* As obtained from Tcl_ZlibStreamInit */ Tcl_Obj *data, /* Data to compress/decompress */ int flush) /* TCL_ZLIB_NO_FLUSH, TCL_ZLIB_FLUSH, * TCL_ZLIB_FULLFLUSH, or TCL_ZLIB_FINALIZE */ { ZlibStreamHandle *zshPtr = (ZlibStreamHandle *) zshandle; char *dataTmp = NULL; | > > | < | 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 | * * Add data to the stream for compression or decompression from a * bytearray Tcl_Obj. * *---------------------------------------------------------------------- */ #define BUFFER_SIZE_LIMIT 0xFFFF int Tcl_ZlibStreamPut( Tcl_ZlibStream zshandle, /* As obtained from Tcl_ZlibStreamInit */ Tcl_Obj *data, /* Data to compress/decompress */ int flush) /* TCL_ZLIB_NO_FLUSH, TCL_ZLIB_FLUSH, * TCL_ZLIB_FULLFLUSH, or TCL_ZLIB_FINALIZE */ { ZlibStreamHandle *zshPtr = (ZlibStreamHandle *) zshandle; char *dataTmp = NULL; int e, size, outSize, toStore; if (zshPtr->streamEnd) { if (zshPtr->interp) { Tcl_SetObjResult(zshPtr->interp, Tcl_NewStringObj( "already past compressed stream end", -1)); Tcl_SetErrorCode(zshPtr->interp, "TCL", "ZIP", "CLOSED", NULL); } |
︙ | ︙ | |||
1171 1172 1173 1174 1175 1176 1177 | if (size == 0) { return TCL_OK; } if (HaveDictToSet(zshPtr)) { e = SetDeflateDictionary(&zshPtr->stream, zshPtr->compDictObj); if (e != Z_OK) { | < | < | | > > | > | > | | > > > > > > > > > > | > | > > > > > > | < | > | < < < | < | < < < < < < < < | < < < | < < < < < < < < | < | 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 | if (size == 0) { return TCL_OK; } if (HaveDictToSet(zshPtr)) { e = SetDeflateDictionary(&zshPtr->stream, zshPtr->compDictObj); if (e != Z_OK) { ConvertError(zshPtr->interp, e, zshPtr->stream.adler); return TCL_ERROR; } DictWasSet(zshPtr); } /* * deflateBound() doesn't seem to take various header sizes into * account, so we add 100 extra bytes. However, we can also loop * around again so we also set an upper bound on the output buffer * size. */ outSize = deflateBound(&zshPtr->stream, size) + 100; if (outSize > BUFFER_SIZE_LIMIT) { outSize = BUFFER_SIZE_LIMIT; } dataTmp = ckalloc(outSize); while (1) { e = Deflate(&zshPtr->stream, dataTmp, outSize, flush, &toStore); /* * Test if we've filled the buffer up and have to ask deflate() to * give us some more. Note that the condition for needing to * repeat a buffer transfer when the result is Z_OK is whether * there is no more space in the buffer we provided; the zlib * library does not necessarily return a different code in that * case. [Bug b26e38a3e4] [Tk Bug 10f2e7872b] */ if ((e != Z_BUF_ERROR) && (e != Z_OK || toStore < outSize)) { if ((e == Z_OK) || (flush == Z_FINISH && e == Z_STREAM_END)) { break; } ConvertError(zshPtr->interp, e, zshPtr->stream.adler); return TCL_ERROR; } /* * Output buffer too small to hold the data being generated or we * are doing the end-of-stream flush (which can spit out masses of * data). This means we need to put a new buffer into place after * saving the old generated data to the outData list. */ AppendByteArray(zshPtr->outData, dataTmp, outSize); if (outSize < BUFFER_SIZE_LIMIT) { outSize = BUFFER_SIZE_LIMIT; /* There may be *lots* of data left to output... */ dataTmp = ckrealloc(dataTmp, outSize); } } /* * And append the final data block to the outData list. */ AppendByteArray(zshPtr->outData, dataTmp, toStore); ckfree(dataTmp); } else { /* * This is easy. Just append to the inData list. */ Tcl_ListObjAppendElement(NULL, zshPtr->inData, data); |
︙ | ︙ | |||
1352 1353 1354 1355 1356 1357 1358 | * (You can't do it in response to getting Z_NEED_DATA as raw streams * don't ever issue that.) */ if (IsRawStream(zshPtr) && HaveDictToSet(zshPtr)) { e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj); if (e != Z_OK) { | < | < | 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 | * (You can't do it in response to getting Z_NEED_DATA as raw streams * don't ever issue that.) */ if (IsRawStream(zshPtr) && HaveDictToSet(zshPtr)) { e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj); if (e != Z_OK) { ConvertError(zshPtr->interp, e, zshPtr->stream.adler); return TCL_ERROR; } DictWasSet(zshPtr); } e = inflate(&zshPtr->stream, zshPtr->flush); if (e == Z_NEED_DICT && HaveDictToSet(zshPtr)) { e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj); |
︙ | ︙ | |||
2875 2876 2877 2878 2879 2880 2881 | /* * Flush any data waiting to be compressed. */ if (cd->mode == TCL_ZLIB_STREAM_DEFLATE) { cd->outStream.avail_in = 0; do { | < < | | | 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 | /* * Flush any data waiting to be compressed. */ if (cd->mode == TCL_ZLIB_STREAM_DEFLATE) { cd->outStream.avail_in = 0; do { e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated, Z_FINISH, &written); /* * Can't be sure that deflate() won't declare the buffer to be * full (with Z_BUF_ERROR) so handle that case. */ if (e == Z_BUF_ERROR) { |
︙ | ︙ | |||
3082 3083 3084 3085 3086 3087 3088 | return outProc(Tcl_GetChannelInstanceData(cd->parent), buf, toWrite, errorCodePtr); } cd->outStream.next_in = (Bytef *) buf; cd->outStream.avail_in = toWrite; do { | < < | | < > | | | 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 | return outProc(Tcl_GetChannelInstanceData(cd->parent), buf, toWrite, errorCodePtr); } cd->outStream.next_in = (Bytef *) buf; cd->outStream.avail_in = toWrite; do { e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated, Z_NO_FLUSH, &produced); if ((e == Z_OK && produced > 0) || e == Z_BUF_ERROR) { /* * deflate() indicates that it is out of space by returning * Z_BUF_ERROR *or* by simply returning Z_OK with no remaining * space; in either case, we must write the whole buffer out and * retry to compress what is left. */ if (e == Z_BUF_ERROR) { produced = cd->outAllocated; e = Z_OK; } if (Tcl_WriteRaw(cd->parent, cd->outBuffer, produced) < 0) { |
︙ | ︙ |
Changes to tests/zlib.test.
︙ | ︙ | |||
134 135 136 137 138 139 140 141 142 143 144 145 146 147 | } {{} 69f34b6a abcdeEDCBA..} test zlib-7.7 {zlib stream: Bug 25842c161} -constraints zlib -body { set s [zlib stream deflate] $s put {} } -cleanup { catch {$s close} } -result "" test zlib-8.1 {zlib transformation} -constraints zlib -setup { set file [makeFile {} test.gz] } -body { set f [zlib push gzip [open $file w] -header {comment gorp}] puts $f "ok" close $f | > > > > > > > > > > > > > > > > > > > | 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | } {{} 69f34b6a abcdeEDCBA..} test zlib-7.7 {zlib stream: Bug 25842c161} -constraints zlib -body { set s [zlib stream deflate] $s put {} } -cleanup { catch {$s close} } -result "" # Also causes Tk Bug 10f2e7872b test zlib-7.8 {zlib stream: Bug b26e38a3e4} -constraints zlib -setup { expr srand(12345) set randdata {} for {set i 0} {$i<6001} {incr i} { append randdata [binary format c [expr {int(256*rand())}]] } } -body { set strm [zlib stream compress] for {set i 1} {$i<3000} {incr i} { $strm put $randdata } $strm put -finalize $randdata set data [$strm get] list [string length $data] [string length [zlib decompress $data]] } -cleanup { catch {$strm close} unset -nocomplain randdata data } -result {120185 18003000} test zlib-8.1 {zlib transformation} -constraints zlib -setup { set file [makeFile {} test.gz] } -body { set f [zlib push gzip [open $file w] -header {comment gorp}] puts $f "ok" close $f |
︙ | ︙ |