expect_memfault/000700 000765 000024 00000000000 12550514055 014154 5ustar00vdhstaff000000 000000 expect_memfault/breakdown000700 000765 000024 00000032425 12550520610 016056 0ustar00vdhstaff000000 000000 #!/usr/bin/env tclsh

#package require math::statistics
#namespace path {::tcl::mathop ::tcl::mathfunc math::statistics}
namespace path {::tcl::mathop}

proc locateDatabases {} {
   upvar env env
   set home $env(HOME)
   set dbdir data
   set searchpath "
      .
      $dbdir
   "
   set hit 0
   foreach dbpath $searchpath {
      if {[file isdirectory $dbpath]} {
         set hit 1
         break
      } 
   }
   if {$hit == 0} {
      puts "could not find the database directory in the list $searchpath"
      exit 1
   }
   if { [catch {glob -directory $dbpath *} databases] } { puts $databases; exit 1 }
   set databases [lsort -dictionary $databases]
   return $databases
}

proc loadDatabase {dbname triptype} {
# ----------------------------------------------------------------
# load a database as a dict whose keys are the names of
# the header fields in the database and where the values are
# the database columns. in a sense this dict thus mimics the usual 'data
# frame' structure: a list of lists containing equal number of elements
# (where empty elements represent missing entries, i.e. 'NAs').
# ----------------------------------------------------------------
   if { [catch {open $dbname} channel] } { puts $channel; exit 1 }
   set buf [read $channel]
   close $channel
   set buf [string trimright $buf]; #... suppress a spurious trailing record
   set buf [split $buf \n]

   set rgxprivate {\(P\)}
   foreach line $buf {
      # first, check for (and skip) comments
      if {[ regexp {^[ ]*\#} $line ]} { continue }
      # locate the header line and extract the field names
      if {[ regexp {^[[:alpha:]]} $line ]} {
         set knownFields [split [join $line]]
         # and don't forget to initialize them in case they
         # are never accessed ...
         foreach field $knownFields { set $field {} }
         continue
      }

      #possibly exclude certain entries from further consideration:
      if {$triptype == "work"} {
         if { [regexp $rgxprivate $line] } { continue }
      } elseif {$triptype == "private"} {
         if { ![regexp $rgxprivate $line] } { continue }
      }

      set record [split $line \t]
      set cnt -1
      foreach field $knownFields {
         lappend $field [lindex $record [incr cnt]]
      }
   }
   foreach field $knownFields {
      dict set df $field [subst $$field]
   }
   return $df
}

proc getLimits alldates {
# ----------------------------------------------------------------
# determine the index ranges for summing over the required time
# frames. strict chronological ordering of `alldates' is presumed.
# ----------------------------------------------------------------
   set now [clock seconds]
   set thisDay [clock scan [clock format $now -format %Y-%m-%d] -format %Y-%m-%d]

   set thisYear [clock format $now -format %Y]
   set lastYear [- $thisYear 1]

   set thisMonth [clock format $now -format %m]
   set lastMonth [format %02d [- $thisMonth 1]]
   if {$lastMonth == 0} {
      set $lastMonth 12
      set thatYear $lastYear
   } else { set thatYear $thisYear }

   # months and years in %Y-%m-%d format:
   set thisMonth $thisYear-$thisMonth-01
   set lastMonth $thatYear-$lastMonth-01
   set thisYear  $thisYear-01-01
   set lastYear  $lastYear-01-01

   # now compute all relevant timepoints in seconds since the epoch.
   # getting this and last week right across year boundaries (i.e. if
   # runnning `breakdown' shortly after New Year's Eve) is achieved by 
   # explicit calculation (leap-seconds not taken into account, though ;-))
   #
   set daylen 86400
   set weekday   [clock format $now -format %u]
   set thisWeek  [expr $thisDay - ($weekday - 1) * $daylen]
   set lastWeek  [expr $thisWeek - 7 * $daylen]
   # the rest is easy:
   foreach period {thisMonth lastMonth thisYear lastYear} {
      set getval [subst $$period]
      set $period [clock scan $getval -format %Y-%m-%d]
   }

   # in general, $binvals is not sorted chronologically so we explicitly
   # establish the sorting order from 'most recent' to 'longer ago' to
   # avoid redundant looping over the database entries when setting up the
   # `chronbins' dict. also note, that the order of the entries in binkeys/binvals
   # must not be changed without adjusting the `stoppoints' determination...
   #
   set binkeys [list "this week :" "last week :" "this month:" "last month:" "this year :" "last year :"]
   set binvals [list  $thisWeek     $lastWeek     $thisMonth    $lastMonth    $thisYear     $lastYear]
   set indices [lsort -decreasing -integer -indices $binvals]
   set chronkeys [lsubset $binkeys $indices] 
   set chronvals [lsubset $binvals $indices] 
   foreach key $chronkeys val $chronvals {
      dict set chronbins $key value $val
      dict set chronbins $key start -1
   }

   # note to self:
   # the following innocuous looking line burns about 94% of all the
   # time in this proc (which in turn is responsible for about 88%
   # of run time of `condense' and 65% of complete run time).
   # lesson learned: `clock scan' on a lot of entries is very very
   # slow... if this script ever becomes too slow (too many entries
   # across all inspected databases) the _only_ way for speed up would
   # be to to do this conversion from dates to seconds not on a per-database
   # basis but once and for all for all dates in the full time range and
   # than implement subsetting into this array for each database. currently
   # conversion is done for each database separately (and thus repeatedly for
   # the (mostly) identical dates).
   #
   set allseconds [lmap date $alldates {clock scan $date -format %Y-%m-%d}]

   # we search backwards in time which should be faster in the long
   # run since we have to scan at most two (and on average one and a
   # half) years back in time. of course, due to the `clock scan' bottle
   # neck mentioned above the speed gain is totallly negligble...
   #
   set allseconds [lreverse $allseconds] 
   set numentries [llength $allseconds]
   
   set last [lindex [dict keys $chronbins] end]
   set idx $numentries 
   foreach entry $allseconds {
      dict for {key val} $chronbins {
         set value [dict get $val value]
         if { [dict get $val start] < 0 && $entry < $value } { dict set chronbins $key start $idx }
         # note to self: if the previous statement triggers, one could elminate that key from
         # future looping ...
      }
      if {[dict get $chronbins $last start] > 0} { break }
      incr idx -1
   }
   # re-sort to achieve one-to-one correspondence to entries in `binvals'
   foreach key $binkeys { lappend start [dict get $chronbins $key start] }
   
   # `stop' is fixed by heuristics: we know that we have alternating 'this/last'
   # entries where 'this' extends to end of range and 'last' extends to previous 'this'
   # minus one. this is ugly but avoids having to search for the stopping points.
   set nbins [llength $binkeys]
   set stop $start
   for {set i 0} {$i < $nbins} {incr i 2} { lset stop $i end }
   for {set i 1} {$i < $nbins} {incr i 2} { lset stop $i [- [lindex $start [- $i 1]] 1] }

   foreach key $binkeys from $start to $stop {
      dict set limits $key start $from 
      dict set limits $key stop $to
   }

   return $limits 
}

proc sumup {alldates alltrips alltimes allrises fromto} {
   set from [dict get $fromto start]
   set to   [dict get $fromto stop]
   if {$from != ""} {
      set dates [lrange $alldates $from $to]
      set trips [lrange $alltrips $from $to]
      set times [lrange $alltimes $from $to]
      set rises [lrange $allrises $from $to]

      # there might be holes (a.k.a. NAs) in `times' and `rises' which need to be removed
      # before trying to sum them up:
      set times [join $times]
      set rises [join $rises]

      set sumdists [+ {*}$trips]
      set sumtrips [llength [lsort -unique $dates]]
      set sumtimes [+ {*}$times]
      set sumtimes [/ $sumtimes 60.]
      set sumrises [+ {*}$rises]
   } else {
      set sumdists 0
      set sumtrips 0
      set sumrises 0
   }
   return [list dists [format %.3f $sumdists] trips $sumtrips rises $sumrises]
}

proc lsubset {x indices} { lmap i $indices {lindex $x $i} }

# memo: this yields a list of indices which permute `from' into `to'
# (presuming that both _are_ permuations of each other...). i.e.
# lsubset $x [lperm $x $y] is identical to $y.
#
## proc lperm {from to} {lmap entry $to {lsearch -exact $from $entry}}

proc analyzeDatabase df {
   set alldates [dict get $df date]
   set alltrips [dict get $df trip]
   set alltimes [dict get $df time]
   set allrises [dict get $df ascent]

   # as long as the databases are kept chronologically sorted re-sorting
   # is redundant. but we want to be on the safe side...
   set idx [lsort -indices $alldates]
   set alldates [lsubset $alldates $idx]
   set alltrips [lsubset $alltrips $idx]
   set alltimes [lsubset $alltimes $idx]
   set allrises [lsubset $allrises $idx]

   set limits [getLimits $alldates]

   dict for {key fromto} $limits {
      set summed [sumup $alldates $alltrips $alltimes $allrises $fromto]
      foreach ctgry {dists trips rises} {
         dict set dbdata $key $ctgry [dict get $summed $ctgry]
      }
   }
   
   set sumall [sumup $alldates $alltrips $alltimes $allrises {start 0 stop end}]
   foreach ctgry {dists trips rises} {
      dict set dbdata "all-time  :" $ctgry [dict get $sumall $ctgry]
   }
   return $dbdata
}

proc condense {databases triptype} {
   set dbaccu {}
   foreach database $databases {
      set df [loadDatabase $database $triptype]
      set dbdata [analyzeDatabase $df]

      if {$dbaccu == ""} {
         set dbaccu $dbdata
      } else {
         foreach key1 [dict keys $dbaccu] {
            foreach key2 [dict keys [dict get $dbaccu $key1]] {
               set val [dict get $dbaccu $key1 $key2]
               lappend val [dict get $dbdata $key1 $key2]
               dict set dbaccu $key1 $key2 $val
            }
         }
      }
      set dbname [lindex [split $database /] end]
      dict set dblist $dbname $dbdata
   }

   foreach key1 [dict keys $dbaccu] {
      foreach key2 [dict keys [dict get $dbaccu $key1]] {
         dict set gtdata $key1 $key2 [+ {*}[dict get $dbaccu $key1 $key2]]
      }
   }

   return [list $dblist $gtdata]
}

proc markup {text styles} {
# ----------------------------------------------
#
#  styles is a 1-2 component list:
#     1. foreground: foreground color or style
#     2. background: background color or style
#
# valid names are those in dictionary `codetable'
# ----------------------------------------------
   set foreground [lindex $styles 0]
   set background [lindex $styles 1]
   set reset "\[0m"

   set codetable {
       normal     0       bright 1      italics  3
       underscore 4       blink  5      reverse  7

       black     30       red   31      green   32
       yellow    33       blue  34      magenta 35       
       cyan      36       grey  37
   }

   if { [catch {dict get $codetable $foreground} fg] } { set fg 0 }
   if { [catch {dict get $codetable $background} bg] } { set bg 0 }
   if {$bg >= 30} { set bg [expr $bg + 10] }

   set code "\[$bg;${fg}m"
   return $code$text$reset
}

proc reportFor {dbname dbdata color sep} {
   lappend ltext $sep
   lappend ltext "\t*** $dbname ***"
   dict for {k1 v1} $dbdata {
      dict with v1 {
         lappend ltext "\t$k1 [format %7.1f $dists] km [format %6d $rises] m [format %3d $trips] d"
      }
   }
   puts [markup [join $ltext \n] $color]
}

proc finalReport {triptype results} {
   lassign $results dblist gtdata

   set gtname together
   set palette "
      d1    blue
      d2      green
      d3     yellow
      d4      red
      $gtname  normal
   "

   set reportwidth 38 ;# ugly hardcoded value, but at least it's approximately equal to 42 ...
   set fullwidth [string length $triptype]
   set halfwidth [/ [- $reportwidth $fullwidth] 2]
   set leftpad   [string repeat { } $halfwidth]
   set rightpad  [string repeat { } [- $reportwidth [string length $leftpad$triptype] 1]]
   set    header {}
   append header $leftpad
   append header [string toupper $triptype]
   append header $rightpad
   puts -nonewline \t[markup $header reverse]

   set sep {}
   dict for {dbname dbdata} $dblist {
      if {[ catch {dict get $palette $dbname} color ]} {
         set color normal ;#fall back to default
      }
      reportFor $dbname $dbdata $color $sep
   }
   set color [dict get $palette $gtname]
   set sep \n\t[string repeat _ [string length $header]]
   reportFor $gtname $gtdata $color $sep
   puts \t[markup $header reverse]
}

proc parse argv {
   package require cmdline

   set options {
      {a   "All trips (work + private)"}
      {p   "Private trips only"}
   }

   set usage "

      SYNOPSIS
         [::cmdline::getArgv0] ?options?

      DESCRIPTION
	 Just another fine script for analysis of our data.

   "

   if {[ catch {::cmdline::getoptions argv $options $usage} optvals ]} {
      puts \n$optvals
      exit 1
   }

   foreach {opt val} $optvals {
      switch $opt {
         a {dict set argvals a $val}
         p {dict set argvals p $val}
      }
   }
   dict with argvals {
      if {$a == 1} {
        set triptype all
      } elseif {$p == 1} {
        set triptype private
      } else { set triptype work }
   }
   return $triptype
}

# ---------------------------------------------------------------
set triptype  [parse $argv]
set databases [locateDatabases] 
set databases {d1 d2 d3 d4}
set results   [condense $databases $triptype] 
finalReport $triptype $results
expect_memfault/d1000600 000765 000024 00000001347 12550515664 014421 0ustar00vdhstaff000000 000000 date    	trip	time	ascent	comment
2013-04-24	34.2		222
2014-03-18	16.6		198
2014-04-02	18.1		22
2014-04-24	18.1		200
2014-04-30	18.1		22
2014-05-14	17.1		200
2014-05-23	17.1		22
2014-06-02	16.6		198
2014-06-06	18.1		22
2014-06-18	16.6		198
2014-07-15	17.1		22
2014-07-25	16.6		198
2014-07-31	17.1		22
2014-08-18	6		0
2014-08-25	17.1		22
2014-09-03	17.1		200
2014-09-18	16.6		198
2014-09-23	18.1		22
2014-10-27	16.6		198
2015-02-16	17.1		200
2015-02-26	17.1		22
2015-04-15	6		0
2015-04-22	3.3		38
2015-05-04	5.0		56
2015-05-08	2.8		38
2015-05-18	2.8		14
2015-05-21	2.8		38
2015-05-26	2.8		14
2015-05-28	2.8		14
2015-06-03	2.8		38
2015-06-05	2.8		14
2015-06-10	2.8		38
2015-06-24	2.9		14
2015-07-01	2.8		38
2015-07-03	2.8		14
2015-07-07	2.8		38
expect_memfault/d2000600 000765 000024 00000003463 12550515664 014423 0ustar00vdhstaff000000 000000 date    	trip	time	ascent	comment
2013-03-28	28.4		272
2013-04-16	28.4		272
2013-04-30	25.8		231
2013-05-14	28.4		272
2013-05-22	27.8		274
2013-06-05	26.2		228
2013-06-14	28.4		272
2013-07-03	27.8		274
2013-07-12	28.4		272
2013-07-31	28.4		272
2013-08-07	13.8		24
2013-08-15	14.6		248
2013-08-21	13.8		24
2013-09-05	14.6		248
2013-09-06	13.8		24
2013-09-16	14	80	248
2013-09-24	14.6		248
2014-01-03	13.8		24
2014-01-08	14.6		248
2014-01-10	13.8		24
2014-01-21	14.6		248
2014-03-03	12.9		26
2014-03-17	14.6		248
2014-03-20	13.8		24
2014-03-27	14.6		248
2014-04-01	13.8		24
2014-04-04	14.6		248
2014-04-08	13.8		24
2014-04-11	14.6		248
2014-04-16	13.8		24
2014-04-28	14.6		248
2014-04-30	13.8		24
2014-05-08	14.6		248
2014-05-19	13.8		24
2014-05-22	12.9		205
2014-05-26	13.8		24
2014-06-05	12.9		205
2014-06-12	13.8		24
2014-07-07	12.9		205
2014-07-14	13.8		24
2014-07-17	14.6		248
2014-07-21	12.9		26
2014-07-24	14.6		248
2014-07-28	13.8		24
2014-08-01	14.6		248
2014-08-05	13.8		24
2014-08-08	14.6		248
2014-08-19	13.8		24
2014-08-22	14.6		248
2014-08-27	13.8		24
2014-09-08	14.6		248
2014-09-10	13.8		24
2014-09-16	14.6		248
2014-09-23	13.8		24
2014-10-06	14.6		248
2014-10-08	12.9		26
2014-10-14	14.6		248
2014-10-21	13.8		24
2014-10-28	14.6		248
2014-11-14	13.8		24
2014-11-21	14.6		248
2014-11-26	13.8		24
2014-12-01	14.6		248
2014-12-03	13.8		24
2014-12-09	14.6		248
2014-12-12	13.8		24
2014-12-17	13.8		24
2015-01-14	14.6		248
2015-01-16	13.8		24
2015-01-29	12.9		205
2015-02-12	12.9		26
2015-02-18	14.6		248
2015-02-23	13.8		24
2015-03-10	14.6		248
2015-03-12	12.9		26
2015-03-19	14.6		248
2015-03-24	13.8		24
2015-04-13	14.6		248
2015-04-15	13.8		24
2015-04-29	14.6		248
2015-05-04	13.8		24
2015-05-07	14.6		248
2015-05-13	13.8		24
2015-06-15	14.6		248
2015-06-17	12.9		26
2015-06-25	14.6		248
2015-06-30	13.8		24
2015-07-07	12.7		205
expect_memfault/d3000600 000765 000024 00000000767 12550515664 014430 0ustar00vdhstaff000000 000000 date    	trip	time	ascent	comment
2013-05-14	11.87	30.83	
2013-05-21	11.66	34.17	
2013-05-23	11.91	29.15	
2013-07-04	12.81	45.73	
2013-07-09	11.68	28.2	
2013-07-15	13.63	44.7	
2013-07-17	11.71	28.02	
2013-07-23	11.52	34.18	
2013-07-25	11.71	27.42	
2013-08-02	11.69	37.9	
2013-08-08	11.96	30.33	
2013-08-16	13.6	45.05	
2013-08-10	41.1	176.8	
2014-04-08	11.9	33.08	
2014-05-08	11.8	36.27	97
2014-05-22	11.9	30.82	
2014-06-20	11.9	35.98	
2014-07-03	11.9	29.68	
2014-07-14	11.4	34.9	
2014-07-16	11.7	30.62	
expect_memfault/d4000600 000765 000024 00000005134 12550515664 014422 0ustar00vdhstaff000000 000000 date    	trip	time	ascent	comment
2013-04-26	24.4		127
2013-05-14	12.2	33.5	97
2013-05-16	12.2	27	30
2013-05-24	10		134
2013-06-07	12.2	32.5	97
2013-06-11	12.2	27.5	30
2013-06-14	12.2	31.5	97
2013-06-18	12.2	26.5	30
2013-06-21	12.2	31	97
2013-06-28	12.2	32.5	97
2013-07-23	12.2	33.5	97
2013-07-25	12.2	25	30
2013-08-02	12.2	33.5	97
2013-08-06	12.2	26.5	30
2013-08-12	12.2	32.5	97
2013-08-14	12.2	27	30
2013-08-19	12.2	32.5	97
2013-08-21	12.2	27.5	30
2013-08-28	12.2	29.5	30
2013-09-02	3.7		
2013-09-07	20.1		
2013-09-23	12.2	32	97
2013-09-30	12.2	27	30
2013-10-05	38.7		150
2013-10-09	12.2	32.5	97
2013-10-16	12.2	30	30
2013-10-19	28.5		
2013-10-23	12.2	33.5	97
2013-10-25	12.2		30
2013-10-30	12.2		97
2013-11-05	11.8		30
2013-11-08	11.8		97
2013-11-13	11.8		30
2014-01-06	12.2		97
2014-01-08	11.8	27	30
2014-01-17	11.8		30
2014-02-11	11.8		30
2014-02-18	12.2		97
2014-02-26	12.2		97
2014-03-03	11.8		30
2014-03-11	12.2		97
2014-03-13	11.8		30
2014-03-18	12.2	32	97
2014-03-20	12.2	28.2	30
2014-04-01	12.2	26.9	30
2014-04-04	12.2	33.7	97
2014-04-08	12.2	31.4	30
2014-04-11	12.2	33.3	97
2014-04-15	12.2	32.2	30
2014-04-22	12.2	27	30
2014-04-25	12.2	34.7	97
2014-04-29	12.2	27.7	30
2014-05-09	12.2	30	30
2014-05-15	12.2	34.5	97
2014-06-02	12.2		30
2014-06-06	 5.1		130
2014-06-10	12.2		30
2014-06-17	12.2		97
2014-06-19	12.2		30
2014-06-24	12.2		97
2014-06-26	12		30
2014-07-01	12.2		97
2014-07-07	12.2		30
2014-07-15	12.2		97
2014-07-17	12.2		30
2014-07-22	12.2		97
2014-07-28	12.2		30
2014-07-31	12.2	32.5	97
2014-08-04	12.2	27.1	30
2014-08-07	12.2	32.6	97
2014-08-11	12.2	31.5	97
2014-08-13	12.2	28.7	30
2014-08-19	12.2	30.9	97
2014-08-21	12.2	29.4	30
2014-08-29	12.2		97
2014-09-03	12.2		30
2014-09-10	22		
2014-09-19	44		
2014-09-24	12.2		30
2014-09-29	12.2		30
2014-10-02	12.2		97
2014-10-06	12.2		97
2014-10-08	12.2	28.17	30
2014-10-27	12.2		97
2014-10-29	11.2	25.98	23
2014-11-06	12.2	35.25	97
2014-11-10	11.2	25.18	23
2014-11-13	11.2	25.75	23
2014-11-27	12.2		97
2015-02-13	11.2	27.5	23
2015-02-26	12.2		97
2015-03-02	11.2		23
2015-03-12	12.2		97
2015-03-24	11.2		23
2015-03-28	39.2		
2015-04-10	12.2	32.67	97
2015-04-14	12.2	32.5	97
2015-04-16	12.2	28.33	30
2015-04-20	12.2	28.25	30
2015-04-25	4.8		
2015-04-29	12.2	33	97
2015-05-04	12.2	25.5	30
2015-05-07	12.2	30.33	97
2015-05-11	12.2	25.67	30
2015-05-18	12.2	32.5	97
2015-05-20	12.2	27	30
2015-05-26	12.2	27	30
2015-05-29	12.2	29.92	97
2015-06-03	12.2	27.42	30
2015-06-08	12.2	32.5	97
2015-06-10	12.2	26.62	30
2015-06-15	12.2	31.5	97
2015-06-17	12.2	26.75	30
2015-06-22	12.2	30.17	97
2015-06-24	12.2		97
2015-06-26	12.2	27	30
2015-07-01	12.2	32.75	97
2015-07-03	12.2		30
2015-07-08	12.2		97
