Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | [b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
f85e0e4fe4f89cf8f24f7b3ec9c7b7b0 |
| User & Date: | dkf 2016-10-30 05:05:46 |
Context
|
2016-10-30
| ||
| 08:31 | [1ae12987cb] Ensure that deleting the [history] command deletes its storage. check-in: eb11bf9cb5 user: dkf tags: trunk | |
| 05:05 | [b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: f85e0e4fe4 user: dkf tags: trunk | |
| 05:04 | [b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: 83a6a4d431 user: dkf tags: core-8-6-branch | |
|
2016-10-28
| ||
| 15:46 | Add optimization to [dict append]. check-in: df0c2fc07c user: dgp tags: trunk | |
Changes
Changes to generic/tclZlib.c.
173 173 static Tcl_ObjCmdProc ZlibStreamAddCmd; 174 174 static Tcl_ObjCmdProc ZlibStreamHeaderCmd; 175 175 static Tcl_ObjCmdProc ZlibStreamPutCmd; 176 176 177 177 static void ConvertError(Tcl_Interp *interp, int code, 178 178 uLong adler); 179 179 static Tcl_Obj * ConvertErrorToList(int code, uLong adler); 180 +static inline int Deflate(z_streamp strm, void *bufferPtr, 181 + int bufferSize, int flush, int *writtenPtr); 180 182 static void ExtractHeader(gz_header *headerPtr, Tcl_Obj *dictObj); 181 183 static int GenerateHeader(Tcl_Interp *interp, Tcl_Obj *dictObj, 182 184 GzipHeader *headerPtr, int *extraSizePtr); 183 185 static int ZlibPushSubcmd(Tcl_Interp *interp, int objc, 184 186 Tcl_Obj *const objv[]); 185 187 static inline int ResultCopy(ZlibChannelData *cd, char *buf, 186 188 int toRead); ................................................................................ 574 576 } 575 577 576 578 if (latin1enc != NULL) { 577 579 Tcl_FreeEncoding(latin1enc); 578 580 } 579 581 } 580 582 583 +/* 584 + * Disentangle the worst of how the zlib API is used. 585 + */ 586 + 581 587 static int 582 588 SetInflateDictionary( 583 589 z_streamp strm, 584 590 Tcl_Obj *compDictObj) 585 591 { 586 592 if (compDictObj != NULL) { 587 593 int length; ................................................................................ 601 607 int length; 602 608 unsigned char *bytes = Tcl_GetByteArrayFromObj(compDictObj, &length); 603 609 604 610 return deflateSetDictionary(strm, bytes, (unsigned) length); 605 611 } 606 612 return Z_OK; 607 613 } 614 + 615 +static inline int 616 +Deflate( 617 + z_streamp strm, 618 + void *bufferPtr, 619 + int bufferSize, 620 + int flush, 621 + int *writtenPtr) 622 +{ 623 + int e; 624 + 625 + strm->next_out = (Bytef *) bufferPtr; 626 + strm->avail_out = (unsigned) bufferSize; 627 + e = deflate(strm, flush); 628 + if (writtenPtr != NULL) { 629 + *writtenPtr = bufferSize - strm->avail_out; 630 + } 631 + return e; 632 +} 633 + 634 +static inline void 635 +AppendByteArray( 636 + Tcl_Obj *listObj, 637 + void *buffer, 638 + int size) 639 +{ 640 + if (size > 0) { 641 + Tcl_Obj *baObj = Tcl_NewByteArrayObj((unsigned char *) buffer, size); 642 + 643 + Tcl_ListObjAppendElement(NULL, listObj, baObj); 644 + } 645 +} 608 646 609 647 /* 610 648 *---------------------------------------------------------------------- 611 649 * 612 650 * Tcl_ZlibStreamInit -- 613 651 * 614 652 * This command initializes a (de)compression context/handle for ................................................................................ 1135 1173 * 1136 1174 * Add data to the stream for compression or decompression from a 1137 1175 * bytearray Tcl_Obj. 1138 1176 * 1139 1177 *---------------------------------------------------------------------- 1140 1178 */ 1141 1179 1180 +#define BUFFER_SIZE_LIMIT 0xFFFF 1181 + 1142 1182 int 1143 1183 Tcl_ZlibStreamPut( 1144 1184 Tcl_ZlibStream zshandle, /* As obtained from Tcl_ZlibStreamInit */ 1145 1185 Tcl_Obj *data, /* Data to compress/decompress */ 1146 1186 int flush) /* TCL_ZLIB_NO_FLUSH, TCL_ZLIB_FLUSH, 1147 1187 * TCL_ZLIB_FULLFLUSH, or TCL_ZLIB_FINALIZE */ 1148 1188 { 1149 1189 ZlibStreamHandle *zshPtr = (ZlibStreamHandle *) zshandle; 1150 1190 char *dataTmp = NULL; 1151 - int e, size, outSize; 1152 - Tcl_Obj *obj; 1191 + int e, size, outSize, toStore; 1153 1192 1154 1193 if (zshPtr->streamEnd) { 1155 1194 if (zshPtr->interp) { 1156 1195 Tcl_SetObjResult(zshPtr->interp, Tcl_NewStringObj( 1157 1196 "already past compressed stream end", -1)); 1158 1197 Tcl_SetErrorCode(zshPtr->interp, "TCL", "ZIP", "CLOSED", NULL); 1159 1198 } ................................................................................ 1171 1210 if (size == 0) { 1172 1211 return TCL_OK; 1173 1212 } 1174 1213 1175 1214 if (HaveDictToSet(zshPtr)) { 1176 1215 e = SetDeflateDictionary(&zshPtr->stream, zshPtr->compDictObj); 1177 1216 if (e != Z_OK) { 1178 - if (zshPtr->interp) { 1179 - ConvertError(zshPtr->interp, e, zshPtr->stream.adler); 1180 - } 1217 + ConvertError(zshPtr->interp, e, zshPtr->stream.adler); 1181 1218 return TCL_ERROR; 1182 1219 } 1183 1220 DictWasSet(zshPtr); 1184 1221 } 1185 1222 1186 1223 /* 1187 - * Deflatebound doesn't seem to take various header sizes into 1188 - * account, so we add 100 extra bytes. 1224 + * deflateBound() doesn't seem to take various header sizes into 1225 + * account, so we add 100 extra bytes. However, we can also loop 1226 + * around again so we also set an upper bound on the output buffer 1227 + * size. 1189 1228 */ 1190 1229 1191 - outSize = deflateBound(&zshPtr->stream, zshPtr->stream.avail_in)+100; 1192 - zshPtr->stream.avail_out = outSize; 1193 - dataTmp = ckalloc(zshPtr->stream.avail_out); 1194 - zshPtr->stream.next_out = (Bytef *) dataTmp; 1230 + outSize = deflateBound(&zshPtr->stream, size) + 100; 1231 + if (outSize > BUFFER_SIZE_LIMIT) { 1232 + outSize = BUFFER_SIZE_LIMIT; 1233 + } 1234 + dataTmp = ckalloc(outSize); 1195 1235 1196 - e = deflate(&zshPtr->stream, flush); 1197 - while (e == Z_BUF_ERROR || (flush == Z_FINISH && e == Z_OK)) { 1236 + while (1) { 1237 + e = Deflate(&zshPtr->stream, dataTmp, outSize, flush, &toStore); 1238 + 1239 + /* 1240 + * Test if we've filled the buffer up and have to ask deflate() to 1241 + * give us some more. Note that the condition for needing to 1242 + * repeat a buffer transfer when the result is Z_OK is whether 1243 + * there is no more space in the buffer we provided; the zlib 1244 + * library does not necessarily return a different code in that 1245 + * case. [Bug b26e38a3e4] [Tk Bug 10f2e7872b] 1246 + */ 1247 + 1248 + if ((e != Z_BUF_ERROR) && (e != Z_OK || toStore < outSize)) { 1249 + if ((e == Z_OK) || (flush == Z_FINISH && e == Z_STREAM_END)) { 1250 + break; 1251 + } 1252 + ConvertError(zshPtr->interp, e, zshPtr->stream.adler); 1253 + return TCL_ERROR; 1254 + } 1255 + 1198 1256 /* 1199 1257 * Output buffer too small to hold the data being generated or we 1200 1258 * are doing the end-of-stream flush (which can spit out masses of 1201 1259 * data). This means we need to put a new buffer into place after 1202 1260 * saving the old generated data to the outData list. 1203 1261 */ 1204 1262 1205 - obj = Tcl_NewByteArrayObj((unsigned char *) dataTmp, outSize); 1206 - Tcl_ListObjAppendElement(NULL, zshPtr->outData, obj); 1263 + AppendByteArray(zshPtr->outData, dataTmp, outSize); 1207 1264 1208 - if (outSize < 0xFFFF) { 1209 - outSize = 0xFFFF; /* There may be *lots* of data left to 1210 - * output... */ 1265 + if (outSize < BUFFER_SIZE_LIMIT) { 1266 + outSize = BUFFER_SIZE_LIMIT; 1267 + /* There may be *lots* of data left to output... */ 1211 1268 dataTmp = ckrealloc(dataTmp, outSize); 1212 1269 } 1213 - zshPtr->stream.avail_out = outSize; 1214 - zshPtr->stream.next_out = (Bytef *) dataTmp; 1215 - 1216 - e = deflate(&zshPtr->stream, flush); 1217 - } 1218 - 1219 - if (e != Z_OK && !(flush==Z_FINISH && e==Z_STREAM_END)) { 1220 - if (zshPtr->interp) { 1221 - ConvertError(zshPtr->interp, e, zshPtr->stream.adler); 1222 - } 1223 - return TCL_ERROR; 1224 1270 } 1225 1271 1226 1272 /* 1227 - * And append the final data block. 1273 + * And append the final data block to the outData list. 1228 1274 */ 1229 1275 1230 - if (outSize - zshPtr->stream.avail_out > 0) { 1231 - obj = Tcl_NewByteArrayObj((unsigned char *) dataTmp, 1232 - outSize - zshPtr->stream.avail_out); 1233 - 1234 - /* 1235 - * Now append the compressed data to the outData list. 1236 - */ 1237 - 1238 - Tcl_ListObjAppendElement(NULL, zshPtr->outData, obj); 1239 - } 1240 - 1241 - if (dataTmp) { 1242 - ckfree(dataTmp); 1243 - } 1276 + AppendByteArray(zshPtr->outData, dataTmp, toStore); 1277 + ckfree(dataTmp); 1244 1278 } else { 1245 1279 /* 1246 1280 * This is easy. Just append to the inData list. 1247 1281 */ 1248 1282 1249 1283 Tcl_ListObjAppendElement(NULL, zshPtr->inData, data); 1250 1284 ................................................................................ 1352 1386 * (You can't do it in response to getting Z_NEED_DATA as raw streams 1353 1387 * don't ever issue that.) 1354 1388 */ 1355 1389 1356 1390 if (IsRawStream(zshPtr) && HaveDictToSet(zshPtr)) { 1357 1391 e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj); 1358 1392 if (e != Z_OK) { 1359 - if (zshPtr->interp) { 1360 - ConvertError(zshPtr->interp, e, zshPtr->stream.adler); 1361 - } 1393 + ConvertError(zshPtr->interp, e, zshPtr->stream.adler); 1362 1394 return TCL_ERROR; 1363 1395 } 1364 1396 DictWasSet(zshPtr); 1365 1397 } 1366 1398 e = inflate(&zshPtr->stream, zshPtr->flush); 1367 1399 if (e == Z_NEED_DICT && HaveDictToSet(zshPtr)) { 1368 1400 e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj); ................................................................................ 2875 2907 /* 2876 2908 * Flush any data waiting to be compressed. 2877 2909 */ 2878 2910 2879 2911 if (cd->mode == TCL_ZLIB_STREAM_DEFLATE) { 2880 2912 cd->outStream.avail_in = 0; 2881 2913 do { 2882 - cd->outStream.next_out = (Bytef *) cd->outBuffer; 2883 - cd->outStream.avail_out = (unsigned) cd->outAllocated; 2884 - e = deflate(&cd->outStream, Z_FINISH); 2885 - written = cd->outAllocated - cd->outStream.avail_out; 2914 + e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated, 2915 + Z_FINISH, &written); 2886 2916 2887 2917 /* 2888 2918 * Can't be sure that deflate() won't declare the buffer to be 2889 2919 * full (with Z_BUF_ERROR) so handle that case. 2890 2920 */ 2891 2921 2892 2922 if (e == Z_BUF_ERROR) { ................................................................................ 3082 3112 return outProc(Tcl_GetChannelInstanceData(cd->parent), buf, toWrite, 3083 3113 errorCodePtr); 3084 3114 } 3085 3115 3086 3116 cd->outStream.next_in = (Bytef *) buf; 3087 3117 cd->outStream.avail_in = toWrite; 3088 3118 do { 3089 - cd->outStream.next_out = (Bytef *) cd->outBuffer; 3090 - cd->outStream.avail_out = cd->outAllocated; 3091 - 3092 - e = deflate(&cd->outStream, Z_NO_FLUSH); 3093 - produced = cd->outAllocated - cd->outStream.avail_out; 3119 + e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated, 3120 + Z_NO_FLUSH, &produced); 3094 3121 3095 3122 if ((e == Z_OK && produced > 0) || e == Z_BUF_ERROR) { 3096 3123 /* 3097 3124 * deflate() indicates that it is out of space by returning 3098 - * Z_BUF_ERROR; in that case, we must write the whole buffer out 3099 - * and retry to compress what is left. 3125 + * Z_BUF_ERROR *or* by simply returning Z_OK with no remaining 3126 + * space; in either case, we must write the whole buffer out and 3127 + * retry to compress what is left. 3100 3128 */ 3101 3129 3102 3130 if (e == Z_BUF_ERROR) { 3103 3131 produced = cd->outAllocated; 3104 3132 e = Z_OK; 3105 3133 } 3106 3134 if (Tcl_WriteRaw(cd->parent, cd->outBuffer, produced) < 0) {
Changes to tests/zlib.test.
134 134 } {{} 69f34b6a abcdeEDCBA..} 135 135 test zlib-7.7 {zlib stream: Bug 25842c161} -constraints zlib -body { 136 136 set s [zlib stream deflate] 137 137 $s put {} 138 138 } -cleanup { 139 139 catch {$s close} 140 140 } -result "" 141 +# Also causes Tk Bug 10f2e7872b 142 +test zlib-7.8 {zlib stream: Bug b26e38a3e4} -constraints zlib -setup { 143 + expr srand(12345) 144 + set randdata {} 145 + for {set i 0} {$i<6001} {incr i} { 146 + append randdata [binary format c [expr {int(256*rand())}]] 147 + } 148 +} -body { 149 + set strm [zlib stream compress] 150 + for {set i 1} {$i<3000} {incr i} { 151 + $strm put $randdata 152 + } 153 + $strm put -finalize $randdata 154 + set data [$strm get] 155 + list [string length $data] [string length [zlib decompress $data]] 156 +} -cleanup { 157 + catch {$strm close} 158 + unset -nocomplain randdata data 159 +} -result {120185 18003000} 141 160 142 161 test zlib-8.1 {zlib transformation} -constraints zlib -setup { 143 162 set file [makeFile {} test.gz] 144 163 } -body { 145 164 set f [zlib push gzip [open $file w] -header {comment gorp}] 146 165 puts $f "ok" 147 166 close $f