Tcl Source Code

Check-in [f85e0e4fe4]
Login
☰Core Home Timeline Branches Tags Tickets Download Wiki

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:[b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f85e0e4fe4f89cf8f24f7b3ec9c7b7b0ce131383
User & Date: dkf 2016-10-30 05:05:46.089
Context
2016-10-30
08:31
[1ae12987cb] Ensure that deleting the [history] command deletes its storage. check-in: eb11bf9cb5 user: dkf tags: trunk
05:05
[b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: f85e0e4fe4 user: dkf tags: trunk
05:04
[b26e38a3e4] Ensure that compressing streams manage buffers correctly in all cases. check-in: 83a6a4d431 user: dkf tags: core-8-6-branch
2016-10-28
15:46
Add optimization to [dict append]. check-in: df0c2fc07c user: dgp tags: trunk
Changes
Side-by-Side Diff Ignore Whitespace Patch
Changes to generic/tclZlib.c.
︙
173
174
175
176
177
178
179


180
181
182
183
184
185
186

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188








+
+








static Tcl_ObjCmdProc		ZlibStreamAddCmd;
static Tcl_ObjCmdProc		ZlibStreamHeaderCmd;
static Tcl_ObjCmdProc		ZlibStreamPutCmd;

static void		ConvertError(Tcl_Interp *interp, int code,
			    uLong adler);
static Tcl_Obj *	ConvertErrorToList(int code, uLong adler);
static inline int	Deflate(z_streamp strm, void *bufferPtr,
			    int bufferSize, int flush, int *writtenPtr);
static void		ExtractHeader(gz_header *headerPtr, Tcl_Obj *dictObj);
static int		GenerateHeader(Tcl_Interp *interp, Tcl_Obj *dictObj,
			    GzipHeader *headerPtr, int *extraSizePtr);
static int		ZlibPushSubcmd(Tcl_Interp *interp, int objc,
			    Tcl_Obj *const objv[]);
static inline int	ResultCopy(ZlibChannelData *cd, char *buf,
			    int toRead);
︙
574
575
576
577
578
579
580




581
582
583
584
585
586
587

576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593








+
+
+
+








    }

    if (latin1enc != NULL) {
	Tcl_FreeEncoding(latin1enc);
    }
}

/*
 * Disentangle the worst of how the zlib API is used.
 */

static int
SetInflateDictionary(
    z_streamp strm,
    Tcl_Obj *compDictObj)
{
    if (compDictObj != NULL) {
	int length;
︙
601
602
603
604
605
606
607
































608
609
610
611
612
613
614

607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








	int length;
	unsigned char *bytes = Tcl_GetByteArrayFromObj(compDictObj, &length);

	return deflateSetDictionary(strm, bytes, (unsigned) length);
    }
    return Z_OK;
}

static inline int
Deflate(
    z_streamp strm,
    void *bufferPtr,
    int bufferSize,
    int flush,
    int *writtenPtr)
{
    int e;

    strm->next_out = (Bytef *) bufferPtr;
    strm->avail_out = (unsigned) bufferSize;
    e = deflate(strm, flush);
    if (writtenPtr != NULL) {
	*writtenPtr = bufferSize - strm->avail_out;
    }
    return e;
}

static inline void
AppendByteArray(
    Tcl_Obj *listObj,
    void *buffer,
    int size)
{
    if (size > 0) {
	Tcl_Obj *baObj = Tcl_NewByteArrayObj((unsigned char *) buffer, size);

	Tcl_ListObjAppendElement(NULL, listObj, baObj);
    }
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_ZlibStreamInit --
 *
 *	This command initializes a (de)compression context/handle for
︙
1135
1136
1137
1138
1139
1140
1141


1142
1143
1144
1145
1146
1147
1148
1149
1150
1151

1152
1153
1154
1155
1156
1157
1158
1159

1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190

1191

1192
1193
1194
1195
1196
1197
1198








+
+









-
+
-








 *
 *	Add data to the stream for compression or decompression from a
 *	bytearray Tcl_Obj.
 *
 *----------------------------------------------------------------------
 */

#define BUFFER_SIZE_LIMIT	0xFFFF

int
Tcl_ZlibStreamPut(
    Tcl_ZlibStream zshandle,	/* As obtained from Tcl_ZlibStreamInit */
    Tcl_Obj *data,		/* Data to compress/decompress */
    int flush)			/* TCL_ZLIB_NO_FLUSH, TCL_ZLIB_FLUSH,
				 * TCL_ZLIB_FULLFLUSH, or TCL_ZLIB_FINALIZE */
{
    ZlibStreamHandle *zshPtr = (ZlibStreamHandle *) zshandle;
    char *dataTmp = NULL;
    int e, size, outSize;
    int e, size, outSize, toStore;
    Tcl_Obj *obj;

    if (zshPtr->streamEnd) {
	if (zshPtr->interp) {
	    Tcl_SetObjResult(zshPtr->interp, Tcl_NewStringObj(
		    "already past compressed stream end", -1));
	    Tcl_SetErrorCode(zshPtr->interp, "TCL", "ZIP", "CLOSED", NULL);
	}
︙
1171
1172
1173
1174
1175
1176
1177
1178
1179

1180
1181
1182
1183
1184
1185
1186
1187
1188




1189
1190
1191
1192
1193
1194








1195








1196
1197









1198
1199
1200
1201
1202
1203
1204
1205

1206
1207
1208
1209



1210
1211
1212
1213
1214
1215

1216
1217

1218
1219
1220
1221
1222
1223
1224
1225
1226
1227

1228
1229
1230
1231
1232
1233

1234
1235
1236
1237
1238
1239
1240
1241
1242

1243
1244
1245
1246
1247
1248
1249
1250

1210
1211
1212
1213
1214
1215
1216


1217

1218
1219
1220
1221
1222
1223


1224
1225
1226
1227
1228
1229




1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246


1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262

1263

1264


1265
1266
1267

1268
1269



1270


1271








1272

1273
1274
1275




1276









1277

1278
1279
1280
1281
1282
1283
1284








-
-
+
-






-
-
+
+
+
+


-
-
-
-
+
+
+
+
+
+
+
+

+
+
+
+
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+







-
+
-

-
-
+
+
+
-


-
-
-
+
-
-
+
-
-
-
-
-
-
-
-

-
+


-
-
-
-
+
-
-
-
-
-
-
-
-
-
+
-








	if (size == 0) {
	    return TCL_OK;
	}

	if (HaveDictToSet(zshPtr)) {
	    e = SetDeflateDictionary(&zshPtr->stream, zshPtr->compDictObj);
	    if (e != Z_OK) {
		if (zshPtr->interp) {
		    ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
		}
		return TCL_ERROR;
	    }
	    DictWasSet(zshPtr);
	}

	/*
	 * Deflatebound doesn't seem to take various header sizes into
	 * account, so we add 100 extra bytes.
	 * deflateBound() doesn't seem to take various header sizes into
	 * account, so we add 100 extra bytes. However, we can also loop
	 * around again so we also set an upper bound on the output buffer
	 * size.
	 */

	outSize = deflateBound(&zshPtr->stream, zshPtr->stream.avail_in)+100;
	zshPtr->stream.avail_out = outSize;
	dataTmp = ckalloc(zshPtr->stream.avail_out);
	zshPtr->stream.next_out = (Bytef *) dataTmp;
	outSize = deflateBound(&zshPtr->stream, size) + 100;
	if (outSize > BUFFER_SIZE_LIMIT) {
	    outSize = BUFFER_SIZE_LIMIT;
	}
	dataTmp = ckalloc(outSize);

	while (1) {
	    e = Deflate(&zshPtr->stream, dataTmp, outSize, flush, &toStore);

	    /*
	     * Test if we've filled the buffer up and have to ask deflate() to
	     * give us some more. Note that the condition for needing to
	     * repeat a buffer transfer when the result is Z_OK is whether
	     * there is no more space in the buffer we provided; the zlib
	     * library does not necessarily return a different code in that
	     * case. [Bug b26e38a3e4] [Tk Bug 10f2e7872b]
	     */
	e = deflate(&zshPtr->stream, flush);
	while (e == Z_BUF_ERROR || (flush == Z_FINISH && e == Z_OK)) {

	    if ((e != Z_BUF_ERROR) && (e != Z_OK || toStore < outSize)) {
		if ((e == Z_OK) || (flush == Z_FINISH && e == Z_STREAM_END)) {
		    break;
		}
		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
		return TCL_ERROR;
	    }

	    /*
	     * Output buffer too small to hold the data being generated or we
	     * are doing the end-of-stream flush (which can spit out masses of
	     * data). This means we need to put a new buffer into place after
	     * saving the old generated data to the outData list.
	     */

	    obj = Tcl_NewByteArrayObj((unsigned char *) dataTmp, outSize);
	    AppendByteArray(zshPtr->outData, dataTmp, outSize);
	    Tcl_ListObjAppendElement(NULL, zshPtr->outData, obj);

	    if (outSize < 0xFFFF) {
		outSize = 0xFFFF;	/* There may be *lots* of data left to
	    if (outSize < BUFFER_SIZE_LIMIT) {
		outSize = BUFFER_SIZE_LIMIT;
		/* There may be *lots* of data left to output... */
					 * output... */
		dataTmp = ckrealloc(dataTmp, outSize);
	    }
	    zshPtr->stream.avail_out = outSize;
	    zshPtr->stream.next_out = (Bytef *) dataTmp;

	}
	    e = deflate(&zshPtr->stream, flush);
	}


	if (e != Z_OK && !(flush==Z_FINISH && e==Z_STREAM_END)) {
	    if (zshPtr->interp) {
		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
	    }
	    return TCL_ERROR;
	}

	/*
	 * And append the final data block.
	 * And append the final data block to the outData list.
	 */

	if (outSize - zshPtr->stream.avail_out > 0) {
	    obj = Tcl_NewByteArrayObj((unsigned char *) dataTmp,
		    outSize - zshPtr->stream.avail_out);

	AppendByteArray(zshPtr->outData, dataTmp, toStore);
	    /*
	     * Now append the compressed data to the outData list.
	     */

	    Tcl_ListObjAppendElement(NULL, zshPtr->outData, obj);
	}

	if (dataTmp) {
	    ckfree(dataTmp);
	ckfree(dataTmp);
	}
    } else {
	/*
	 * This is easy. Just append to the inData list.
	 */

	Tcl_ListObjAppendElement(NULL, zshPtr->inData, data);
︙
1352
1353
1354
1355
1356
1357
1358
1359
1360

1361
1362
1363
1364
1365
1366
1367
1368

1386
1387
1388
1389
1390
1391
1392


1393

1394
1395
1396
1397
1398
1399
1400








-
-
+
-








	 * (You can't do it in response to getting Z_NEED_DATA as raw streams
	 * don't ever issue that.)
	 */

	if (IsRawStream(zshPtr) && HaveDictToSet(zshPtr)) {
	    e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj);
	    if (e != Z_OK) {
		if (zshPtr->interp) {
		    ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
		ConvertError(zshPtr->interp, e, zshPtr->stream.adler);
		}
		return TCL_ERROR;
	    }
	    DictWasSet(zshPtr);
	}
	e = inflate(&zshPtr->stream, zshPtr->flush);
	if (e == Z_NEED_DICT && HaveDictToSet(zshPtr)) {
	    e = SetInflateDictionary(&zshPtr->stream, zshPtr->compDictObj);
︙
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885


2886
2887
2888
2889
2890
2891
2892

2907
2908
2909
2910
2911
2912
2913




2914
2915
2916
2917
2918
2919
2920
2921
2922








-
-
-
-
+
+








    /*
     * Flush any data waiting to be compressed.
     */

    if (cd->mode == TCL_ZLIB_STREAM_DEFLATE) {
	cd->outStream.avail_in = 0;
	do {
	    cd->outStream.next_out = (Bytef *) cd->outBuffer;
	    cd->outStream.avail_out = (unsigned) cd->outAllocated;
	    e = deflate(&cd->outStream, Z_FINISH);
	    written = cd->outAllocated - cd->outStream.avail_out;
	    e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated,
		    Z_FINISH, &written);

	    /*
	     * Can't be sure that deflate() won't declare the buffer to be
	     * full (with Z_BUF_ERROR) so handle that case.
	     */

	    if (e == Z_BUF_ERROR) {
︙
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092


3093
3094
3095
3096
3097

3098
3099


3100
3101
3102
3103
3104
3105
3106

3112
3113
3114
3115
3116
3117
3118




3119
3120

3121
3122
3123
3124
3125


3126
3127
3128
3129
3130
3131
3132
3133
3134








-
-
-
-
+
+
-




+
-
-
+
+








	return outProc(Tcl_GetChannelInstanceData(cd->parent), buf, toWrite,
		errorCodePtr);
    }

    cd->outStream.next_in = (Bytef *) buf;
    cd->outStream.avail_in = toWrite;
    do {
	cd->outStream.next_out = (Bytef *) cd->outBuffer;
	cd->outStream.avail_out = cd->outAllocated;

	e = deflate(&cd->outStream, Z_NO_FLUSH);
	e = Deflate(&cd->outStream, cd->outBuffer, cd->outAllocated,
		Z_NO_FLUSH, &produced);
	produced = cd->outAllocated - cd->outStream.avail_out;

	if ((e == Z_OK && produced > 0) || e == Z_BUF_ERROR) {
	    /*
	     * deflate() indicates that it is out of space by returning
	     * Z_BUF_ERROR *or* by simply returning Z_OK with no remaining
	     * Z_BUF_ERROR; in that case, we must write the whole buffer out
	     * and retry to compress what is left.
	     * space; in either case, we must write the whole buffer out and
	     * retry to compress what is left.
	     */

	    if (e == Z_BUF_ERROR) {
		produced = cd->outAllocated;
		e = Z_OK;
	    }
	    if (Tcl_WriteRaw(cd->parent, cd->outBuffer, produced) < 0) {
︙
Changes to tests/zlib.test.
︙
134
135
136
137
138
139
140



















141
142
143
144
145
146
147

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








} {{} 69f34b6a abcdeEDCBA..}
test zlib-7.7 {zlib stream: Bug 25842c161} -constraints zlib -body {
    set s [zlib stream deflate]
    $s put {}
} -cleanup {
    catch {$s close}
} -result ""
# Also causes Tk Bug 10f2e7872b
test zlib-7.8 {zlib stream: Bug b26e38a3e4} -constraints zlib -setup {
    expr srand(12345)
    set randdata {}
    for {set i 0} {$i<6001} {incr i} {
	append randdata [binary format c [expr {int(256*rand())}]]
    }
} -body {
    set strm [zlib stream compress]
    for {set i 1} {$i<3000} {incr i} {
	$strm put $randdata
    }
    $strm put -finalize $randdata
    set data [$strm get]
    list [string length $data] [string length [zlib decompress $data]]
} -cleanup {
    catch {$strm close}
    unset -nocomplain randdata data
} -result {120185 18003000}

test zlib-8.1 {zlib transformation} -constraints zlib -setup {
    set file [makeFile {} test.gz]
} -body {
    set f [zlib push gzip [open $file w] -header {comment gorp}]
    puts $f "ok"
    close $f
︙
This page was generated in about 0.028s by Fossil 2.27 [a6d506ecc4] 2025-07-09 13:52:53