Index: generic/tclIO.c ================================================================== --- generic/tclIO.c +++ generic/tclIO.c @@ -4358,15 +4358,19 @@ if (srcLen) { WillWrite(chanPtr); } /* - * Transfer encoding strict option to the encoding flags + * Transfer encoding strict/nocomplain option to the encoding flags */ + if (GotFlag(statePtr, CHANNEL_ENCODING_STRICT)) { statePtr->outputEncodingFlags |= TCL_ENCODING_STRICT; + } else if (GotFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN)) { + statePtr->outputEncodingFlags &= ~TCL_ENCODING_STRICT; + statePtr->outputEncodingFlags |= TCL_ENCODING_NOCOMPLAIN; } else { statePtr->outputEncodingFlags &= ~TCL_ENCODING_STRICT; } /* @@ -4405,11 +4409,11 @@ bufPtr->nextAdded += saved; saved = 0; } dst = InsertPoint(bufPtr); dstLen = SpaceLeft(bufPtr); - + result = Tcl_UtfToExternal(NULL, encoding, src, srcLimit, statePtr->outputEncodingFlags, &statePtr->outputEncodingState, dst, dstLen + BUFFER_PADDING, &srcRead, &dstWrote, NULL); @@ -4685,15 +4689,18 @@ if (encoding == NULL) { encoding = GetBinaryEncoding(); } /* - * Transfer encoding strict option to the encoding flags + * Transfer encoding nocomplain/strict option to the encoding flags */ if (GotFlag(statePtr, CHANNEL_ENCODING_STRICT)) { statePtr->inputEncodingFlags |= TCL_ENCODING_STRICT; + } else if (GotFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN)) { + statePtr->inputEncodingFlags &= ~TCL_ENCODING_STRICT; + statePtr->inputEncodingFlags |= TCL_ENCODING_NOCOMPLAIN; } else { statePtr->inputEncodingFlags &= ~TCL_ENCODING_STRICT; } /* @@ -5453,15 +5460,18 @@ *gsPtr->dstPtr = dst; } gsPtr->state = statePtr->inputEncodingState; /* - * Transfer encoding strict option to the encoding flags + * Transfer encoding nocomplain/strict option to the encoding flags */ if (GotFlag(statePtr, CHANNEL_ENCODING_STRICT)) { statePtr->inputEncodingFlags |= TCL_ENCODING_STRICT; + } else if (GotFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN)) { + statePtr->inputEncodingFlags &= ~TCL_ENCODING_STRICT; + statePtr->inputEncodingFlags |= TCL_ENCODING_NOCOMPLAIN; } else { statePtr->inputEncodingFlags &= ~TCL_ENCODING_STRICT; } result = Tcl_ExternalToUtf(NULL, gsPtr->encoding, raw, rawLen, @@ -6236,15 +6246,18 @@ } else { dst = TclGetString(objPtr) + numBytes; } /* - * Transfer encoding strict option to the encoding flags + * Transfer encoding nocomplain/strict option to the encoding flags */ if (GotFlag(statePtr, CHANNEL_ENCODING_STRICT)) { statePtr->inputEncodingFlags |= TCL_ENCODING_STRICT; + } else if (GotFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN)) { + statePtr->inputEncodingFlags &= ~TCL_ENCODING_STRICT; + statePtr->inputEncodingFlags |= TCL_ENCODING_NOCOMPLAIN; } else { statePtr->inputEncodingFlags &= ~TCL_ENCODING_STRICT; } /* @@ -7974,10 +7987,20 @@ Tcl_DStringEndSublist(dsPtr); } if (len > 0) { return TCL_OK; } + } + if (len == 0 || HaveOpt(1, "-nocomplainencoding")) { + if (len == 0) { + Tcl_DStringAppendElement(dsPtr, "-nocomplainencoding"); + } + Tcl_DStringAppendElement(dsPtr, + (flags & CHANNEL_ENCODING_NOCOMPLAIN) ? "1" : "0"); + if (len > 0) { + return TCL_OK; + } } if (len == 0 || HaveOpt(1, "-strictencoding")) { if (len == 0) { Tcl_DStringAppendElement(dsPtr, "-strictencoding"); } @@ -8247,18 +8270,46 @@ if (GotFlag(statePtr, CHANNEL_EOF)) { statePtr->inputEncodingFlags |= TCL_ENCODING_START; } ResetFlag(statePtr, CHANNEL_EOF|CHANNEL_STICKY_EOF|CHANNEL_BLOCKED); statePtr->inputEncodingFlags &= ~TCL_ENCODING_END; + return TCL_OK; + } else if (HaveOpt(1, "-nocomplainencoding")) { + int newMode; + + if (Tcl_GetBoolean(interp, newValue, &newMode) == TCL_ERROR) { + return TCL_ERROR; + } + if (newMode) { + if (GotFlag(statePtr, CHANNEL_ENCODING_STRICT)) { + if (interp) { + Tcl_SetObjResult(interp, Tcl_NewStringObj( + "-nocomplainencoding cannot be used with -strictencoding", + -1)); + } + return TCL_ERROR; + } + SetFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN); + } else { + ResetFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN); + } return TCL_OK; } else if (HaveOpt(1, "-strictencoding")) { int newMode; if (Tcl_GetBoolean(interp, newValue, &newMode) == TCL_ERROR) { return TCL_ERROR; } if (newMode) { + if (GotFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN)) { + if (interp) { + Tcl_SetObjResult(interp, Tcl_NewStringObj( + "-strictencoding cannot be used with -nocomplainencoding", + -1)); + } + return TCL_ERROR; + } SetFlag(statePtr, CHANNEL_ENCODING_STRICT); } else { ResetFlag(statePtr, CHANNEL_ENCODING_STRICT); } return TCL_OK; Index: generic/tclIO.h ================================================================== --- generic/tclIO.h +++ generic/tclIO.h @@ -271,10 +271,12 @@ * delivered for buffered data until * the state of the channel * changes. */ #define CHANNEL_RAW_MODE (1<<16) /* When set, notes that the Raw API is * being used. */ +#define CHANNEL_ENCODING_NOCOMPLAIN (1<<17) /* set if option + * -nocomplainencoding is set to 1 */ #define CHANNEL_ENCODING_STRICT (1<<18) /* set if option * -strictencoding is set to 1 */ #define CHANNEL_INCLOSE (1<<19) /* Channel is currently being closed. * Its structures are still live and * usable, but it may not be closed Index: tests/io.test ================================================================== --- tests/io.test +++ tests/io.test @@ -8950,15 +8950,102 @@ interp delete child testobj freeallvars removeFile io-74.1 } -returnCodes error -match glob -result {can not find channel named "*"} -# The following tests 75.1 to 75.5 exercise strict or tolerant channel -# encoding. -# They are left as a place-holder here. If TIP633 is voted, they will -# come back. -# Exercise strct channel encoding +test io-75.1 {multibyte encoding error read results in raw bytes (-nocomplainencoding 1)} -setup { + set fn [makeFile {} io-75.1] + set f [open $fn w+] + fconfigure $f -encoding binary + # In UTF-8, a byte 0xCx starts a multibyte sequence and must be followed + # by a byte > 0x7F. This is violated to get an invalid sequence. + puts -nonewline $f "A\xC0\x40" + flush $f + seek $f 0 + fconfigure $f -encoding utf-8 -nocomplainencoding 1 -buffering none +} -body { + set d [read $f] + binary scan $d H* hd + set hd +} -cleanup { + close $f + removeFile io-75.1 +} -result "41c040" + +test io-75.2 {unrepresentable character write passes and is replaced by ? (-nocomplainencoding 1)} -setup { + set fn [makeFile {} io-75.2] + set f [open $fn w+] + fconfigure $f -encoding iso8859-1 -nocomplainencoding 1 +} -body { + puts -nonewline $f "A\u2022" + flush $f + seek $f 0 + read $f +} -cleanup { + close $f + removeFile io-75.2 +} -result "A?" + +# Incomplete sequence test. +# This error may IMHO only be detected with the close. +# But the read already returns the incomplete sequence. +test io-75.3 {incomplete multibyte encoding read is ignored (-nocomplainencoding 1)} -setup { + set fn [makeFile {} io-75.3] + set f [open $fn w+] + fconfigure $f -encoding binary + puts -nonewline $f "A\xC0" + flush $f + seek $f 0 + fconfigure $f -encoding utf-8 -buffering none -nocomplainencoding 1 +} -body { + set d [read $f] + close $f + binary scan $d H* hd + set hd +} -cleanup { + removeFile io-75.3 +} -result "41c0" + +# As utf-8 has a special treatment in multi-byte decoding, also test another +# one. +test io-75.4 {shiftjis encoding error read results in raw bytes (-nocomplainencoding 1)} -setup { + set fn [makeFile {} io-75.4] + set f [open $fn w+] + fconfigure $f -encoding binary + # In shiftjis, \x81 starts a two-byte sequence. + # But 2nd byte \xFF is not allowed + puts -nonewline $f "A\x81\xFFA" + flush $f + seek $f 0 + fconfigure $f -encoding shiftjis -buffering none -eofchar "" -translation lf -nocomplainencoding 1 +} -body { + set d [read $f] + binary scan $d H* hd + set hd +} -cleanup { + close $f + removeFile io-75.4 +} -result "4181ff41" + +test io-75.5 {incomplete shiftjis encoding read is ignored (-nocomplainencoding 1)} -setup { + set fn [makeFile {} io-75.5] + set f [open $fn w+] + fconfigure $f -encoding binary + # \x81 announces a two byte sequence. + puts -nonewline $f "A\x81" + flush $f + seek $f 0 + fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -nocomplainencoding 1 +} -body { + set d [read $f] + close $f + binary scan $d H* hd + set hd +} -cleanup { + removeFile io-75.5 +} -result "4181" + test io-75.6 {multibyte encoding error read results in raw bytes} -setup { set fn [makeFile {} io-75.6] set f [open $fn w+] fconfigure $f -encoding binary # In UTF-8, a byte 0xCx starts a multibyte sequence and must be followed Index: tests/ioCmd.test ================================================================== --- tests/ioCmd.test +++ tests/ioCmd.test @@ -243,11 +243,11 @@ set f1 [open $path(test1) w] fconfigure $f1 -translation lf -eofchar {} -encoding utf-16 fconfigure $f1 } -cleanup { catch {close $f1} -} -result {-blocking 1 -buffering full -buffersize 4096 -encoding utf-16 -eofchar {} -strictencoding 0 -translation lf} +} -result {-blocking 1 -buffering full -buffersize 4096 -encoding utf-16 -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf} test iocmd-8.8 {fconfigure command} -setup { file delete $path(test1) set x {} } -body { set f1 [open $path(test1) w] @@ -255,21 +255,21 @@ -eofchar {} -encoding utf-16 lappend x [fconfigure $f1 -buffering] lappend x [fconfigure $f1] } -cleanup { catch {close $f1} -} -result {line {-blocking 1 -buffering line -buffersize 3030 -encoding utf-16 -eofchar {} -strictencoding 0 -translation lf}} +} -result {line {-blocking 1 -buffering line -buffersize 3030 -encoding utf-16 -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf}} test iocmd-8.9 {fconfigure command} -setup { file delete $path(test1) } -body { set f1 [open $path(test1) w] fconfigure $f1 -translation binary -buffering none -buffersize 4040 \ -eofchar {} -encoding binary fconfigure $f1 } -cleanup { catch {close $f1} -} -result {-blocking 1 -buffering none -buffersize 4040 -encoding binary -eofchar {} -strictencoding 0 -translation lf} +} -result {-blocking 1 -buffering none -buffersize 4040 -encoding binary -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf} test iocmd-8.10 {fconfigure command} -returnCodes error -body { fconfigure a b } -result {can not find channel named "a"} set path(fconfigure.dummy) [makeFile {} fconfigure.dummy] test iocmd-8.11 {fconfigure command} -body { @@ -1361,20 +1361,20 @@ set c [chan create {r w} foo] note [fconfigure $c] close $c rename foo {} set res -} -result {{-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -strictencoding 0 -translation {auto *}}} +} -result {{-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -nocomplainencoding 0 -strictencoding 0 -translation {auto *}}} test iocmd-25.2 {chan configure, cgetall, no options} -match glob -body { set res {} proc foo {args} {oninit cget cgetall; onfinal; track; return ""} set c [chan create {r w} foo] note [fconfigure $c] close $c rename foo {} set res -} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -strictencoding 0 -translation {auto *}}} +} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -nocomplainencoding 0 -strictencoding 0 -translation {auto *}}} test iocmd-25.3 {chan configure, cgetall, regular result} -match glob -body { set res {} proc foo {args} { oninit cget cgetall; onfinal; track return "-bar foo -snarf x" @@ -1382,11 +1382,11 @@ set c [chan create {r w} foo] note [fconfigure $c] close $c rename foo {} set res -} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -strictencoding 0 -translation {auto *} -bar foo -snarf x}} +} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -nocomplainencoding 0 -strictencoding 0 -translation {auto *} -bar foo -snarf x}} test iocmd-25.4 {chan configure, cgetall, bad result, list of uneven length} -match glob -body { set res {} proc foo {args} { oninit cget cgetall; onfinal; track return "-bar" Index: tests/socket.test ================================================================== --- tests/socket.test +++ tests/socket.test @@ -1069,11 +1069,11 @@ set s [socket -server accept -myaddr $localhost 0] set l [fconfigure $s] close $s update llength $l -} -result 16 +} -result 18 test socket_$af-7.4 {testing socket specific options} -constraints [list socket supported_$af] -setup { set timer [after 10000 "set x timed_out"] set l "" } -body { set s [socket -server accept -myaddr $localhost 0] Index: tests/zlib.test ================================================================== --- tests/zlib.test +++ tests/zlib.test @@ -290,21 +290,21 @@ list [fconfigure $fd] [zlib push compress $fd; fconfigure $fd] \ [chan pop $fd; fconfigure $fd] } -cleanup { catch {close $fd} removeFile $file -} -result {{-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -strictencoding 0 -translation lf} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -strictencoding 0 -translation lf -checksum 1 -dictionary {}} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -strictencoding 0 -translation lf}} +} -result {{-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf -checksum 1 -dictionary {}} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf}} test zlib-8.7 {transformation and fconfigure} -setup { set file [makeFile {} test.gz] set fd [open $file wb] } -constraints zlib -body { list [fconfigure $fd] [zlib push gzip $fd; fconfigure $fd] \ [chan pop $fd; fconfigure $fd] } -cleanup { catch {close $fd} removeFile $file -} -result {{-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -strictencoding 0 -translation lf} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -strictencoding 0 -translation lf -checksum 0} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -strictencoding 0 -translation lf}} +} -result {{-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf -checksum 0} {-blocking 1 -buffering full -buffersize 4096 -encoding binary -eofchar {} -nocomplainencoding 0 -strictencoding 0 -translation lf}} # Input is headers from fetching SPDY draft # Dictionary is that which is proposed _in_ SPDY draft set spdyHeaders "HTTP/1.0 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nX-Robots-Tag: noarchive\r\nLast-Modified: Tue, 05 Jun 2012 02:43:25 GMT\r\nETag: \"1338864205129|#public|0|en|||0\"\r\nExpires: Tue, 05 Jun 2012 16:17:11 GMT\r\nDate: Tue, 05 Jun 2012 16:17:06 GMT\r\nCache-Control: public, max-age=5\r\nX-Content-Type-Options: nosniff\r\nX-XSS-Protection: 1; mode=block\r\nServer: GSE\r\n" set spdyDict "optionsgetheadpostputdeletetraceacceptaccept-charsetaccept-encodingaccept-languageauthorizationexpectfromhostif-modified-sinceif-matchif-none-matchif-rangeif-unmodifiedsincemax-forwardsproxy-authorizationrangerefererteuser-agent100101200201202203204205206300301302303304305306307400401402403404405406407408409410411412413414415416417500501502503504505accept-rangesageetaglocationproxy-authenticatepublicretry-afterservervarywarningwww-authenticateallowcontent-basecontent-encodingcache-controlconnectiondatetrailertransfer-encodingupgradeviawarningcontent-languagecontent-lengthcontent-locationcontent-md5content-rangecontent-typeetagexpireslast-modifiedset-cookieMondayTuesdayWednesdayThursdayFridaySaturdaySundayJanFebMarAprMayJunJulAugSepOctNovDecchunkedtext/htmlimage/pngimage/jpgimage/gifapplication/xmlapplication/xhtmltext/plainpublicmax-agecharset=iso-8859-1utf-8gzipdeflateHTTP/1.1statusversionurl" test zlib-8.8 {transformation and fconfigure} -setup {