add vendoring with go dep

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/README.md

@@ -0,0 +1,152 @@
+# Converting b1 and bz1 shards to tsm1
+
+`influx_tsm` is a tool for converting b1 and bz1 shards to tsm1
+format. Converting shards to tsm1 format results in a very significant
+reduction in disk usage, and significantly improved write-throughput,
+when writing data into those shards.
+
+Conversion can be controlled on a database-by-database basis. By
+default a database is backed up before it is converted, allowing you
+to roll back any changes. Because of the backup process, ensure the
+host system has at least as much free disk space as the disk space
+consumed by the _data_ directory of your InfluxDB system.
+
+The tool automatically ignores tsm1 shards, and can be run
+idempotently on any database.
+
+Conversion is an offline process, and the InfluxDB system must be
+stopped during conversion. However the conversion process reads and
+writes shards directly on disk and should be fast.
+
+## Steps
+
+Follow these steps to perform a conversion.
+
+* Identify the databases you wish to convert. You can convert one or more databases at a time. By default all databases are converted.
+* Decide on parallel operation. By default the conversion operation peforms each operation in a serial manner. This minimizes load on the host system performing the conversion, but also takes the most time. If you wish to minimize the time conversion takes, enable parallel mode. Conversion will then perform as many operations as possible in parallel, but the process may place significant load on the host system (CPU, disk, and RAM, usage will all increase).
+* Stop all write-traffic to your InfluxDB system.
+* Restart the InfluxDB service and wait until all WAL data is flushed to disk -- this has completed when the system responds to queries. This is to ensure all data is present in shards.
+* Stop the InfluxDB service. It should not be restarted until conversion is complete.
+* Run conversion tool. Depending on the size of the data directory, this might be a lengthy operation. Consider running the conversion tool under a "screen" session to avoid any interruptions.
+* Unless you ran the conversion tool as the same user as that which runs InfluxDB, then you may need to set the correct read-and-write permissions on the new tsm1 directories.
+* Restart node and ensure data looks correct.
+* If everything looks OK, you may then wish to remove or archive the backed-up databases.
+* Restart write traffic.
+
+## Example session
+
+Below is an example session, showing a database being converted.
+
+```
+$ # Create a backup location that the `influxdb` user has full access to
+$ mkdir -m 0777 /path/to/influxdb_backup
+$ sudo -u influxdb influx_tsm -backup /path/to/influxdb_backup -parallel /var/lib/influxdb/data
+
+b1 and bz1 shard conversion.
+-----------------------------------
+Data directory is:                  /var/lib/influxdb/data
+Backup directory is:                /path/to/influxdb_backup
+Databases specified:                all
+Database backups enabled:           yes
+Parallel mode enabled (GOMAXPROCS): yes (8)
+
+
+Found 1 shards that will be converted.
+
+Database        Retention       Path                                                    Engine  Size
+_internal       monitor         /var/lib/influxdb/data/_internal/monitor/1           bz1     65536
+
+These shards will be converted. Proceed? y/N: y
+Conversion starting....
+Backing up 1 databases...
+2016/01/28 12:23:43.699266 Backup of databse '_internal' started
+2016/01/28 12:23:43.699883 Backing up file /var/lib/influxdb/data/_internal/monitor/1
+2016/01/28 12:23:43.700052 Database _internal backed up (851.776µs)
+2016/01/28 12:23:43.700320 Starting conversion of shard: /var/lib/influxdb/data/_internal/monitor/1
+2016/01/28 12:23:43.706276 Conversion of /var/lib/influxdb/data/_internal/monitor/1 successful (6.040148ms)
+
+Summary statistics
+========================================
+Databases converted:                 1
+Shards converted:                    1
+TSM files created:                   1
+Points read:                         369
+Points written:                      369
+NaN filtered:                        0
+Inf filtered:                        0
+Points without fields filtered:      0
+Disk usage pre-conversion (bytes):   65536
+Disk usage post-conversion (bytes):  11000
+Reduction factor:                    83%
+Bytes per TSM point:                 29.81
+Total conversion time:               7.330443ms
+
+$ # restart node, verify data
+$ sudo rm -r /path/to/influxdb_backup
+```
+
+Note that the tool first lists the shards that will be converted,
+before asking for confirmation. You can abort the conversion process
+at this step if you just wish to see what would be converted, or if
+the list of shards does not look correct.
+
+__WARNING:__ If you run the `influx_tsm` tool as a user other than the
+`influxdb` user (or the user that the InfluxDB process runs under),
+please make sure to verify the shard permissions are correct prior to
+starting InfluxDB. If needed, shard permissions can be corrected with
+the `chown` command. For example:
+
+```
+sudo chown -R influxdb:influxdb /var/lib/influxdb
+```
+
+## Rolling back a conversion
+
+After a successful backup (the message `Database XYZ backed up` was
+logged), you have a duplicate of that database in the _backup_
+directory you provided on the command line. If, when checking your
+data after a successful conversion, you notice things missing or
+something just isn't right, you can "undo" the conversion:
+
+- Shut down your node (this is very important)
+- Remove the database's directory from the influxdb `data` directory (default: `~/.influxdb/data/XYZ` for binary installations or `/var/lib/influxdb/data/XYZ` for packaged installations)
+- Copy (to really make sure the shard is preserved) the database's directory from the backup directory you created into the `data` directory.
+
+Using the same directories as above, and assuming a database named `stats`:
+
+```
+$ sudo rm -r /var/lib/influxdb/data/stats
+$ sudo cp -r /path/to/influxdb_backup/stats /var/lib/influxdb/data/
+$ # restart influxd node
+```
+
+#### How to avoid downtime when upgrading shards
+
+*Identify non-`tsm1` shards*
+
+Non-`tsm1` shards are files of the form: `data/<database>/<retention_policy>/<shard_id>`.
+
+`tsm1` shards are files of the form: `data/<database>/<retention_policy>/<shard_id>/<file>.tsm`.
+
+*Determine which `bz`/`bz1` shards are cold for writes*
+
+Run the `SHOW SHARDS` query to see the start and end dates for shards.
+If the date range for a shard does not span the current time then the shard is said to be cold for writes.
+This means that no new points are expected to be added to the shard.
+The shard whose date range spans now is said to be hot for writes.
+You can only safely convert cold shards without stopping the InfluxDB process.
+
+*Convert cold shards*
+
+1. Copy each of the cold shards you'd like to convert to a new directory with the structure `/tmp/data/<database>/<retention_policy>/<shard_id>`.
+2. Run the `influx_tsm` tool on the copied files:
+```
+influx_tsm -parallel /tmp/data/
+```
+3. Remove the existing cold `b1`/`bz1` shards from the production data directory.
+4. Move the new `tsm1` shards into the original directory, overwriting the existing `b1`/`bz1` shards of the same name. Do this simultaneously with step 3 to avoid any query errors.
+5. Wait an hour, a day, or a week (depending on your retention period) for any hot `b1`/`bz1` shards to become cold and repeat steps 1 through 4 on the newly cold shards.
+
+> **Note:** Any points written to the cold shards after making a copy will be lost when the `tsm1` shard overwrites the existing cold shard.
+Nothing in InfluxDB will prevent writes to cold shards, they are merely unexpected, not impossible.
+It is your responsibility to prevent writes to cold shards to prevent data loss.

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/b1/reader.go

@@ -0,0 +1,270 @@
+// Package b1 reads data from b1 shards.
+package b1 // import "github.com/influxdata/influxdb/cmd/influx_tsm/b1"
+
+import (
+	"encoding/binary"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// DefaultChunkSize is the size of chunks read from the b1 shard
+const DefaultChunkSize int = 1000
+
+var excludedBuckets = map[string]bool{
+	"fields": true,
+	"meta":   true,
+	"series": true,
+	"wal":    true,
+}
+
+// Reader is used to read all data from a b1 shard.
+type Reader struct {
+	path string
+	db   *bolt.DB
+	tx   *bolt.Tx
+
+	cursors    []*cursor
+	currCursor int
+
+	keyBuf   string
+	values   []tsm1.Value
+	valuePos int
+
+	fields map[string]*tsdb.MeasurementFields
+	codecs map[string]*tsdb.FieldCodec
+
+	stats *stats.Stats
+}
+
+// NewReader returns a reader for the b1 shard at path.
+func NewReader(path string, stats *stats.Stats, chunkSize int) *Reader {
+	r := &Reader{
+		path:   path,
+		fields: make(map[string]*tsdb.MeasurementFields),
+		codecs: make(map[string]*tsdb.FieldCodec),
+		stats:  stats,
+	}
+
+	if chunkSize <= 0 {
+		chunkSize = DefaultChunkSize
+	}
+
+	r.values = make([]tsm1.Value, chunkSize)
+
+	return r
+}
+
+// Open opens the reader.
+func (r *Reader) Open() error {
+	// Open underlying storage.
+	db, err := bolt.Open(r.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	r.db = db
+
+	// Load fields.
+	if err := r.db.View(func(tx *bolt.Tx) error {
+		meta := tx.Bucket([]byte("fields"))
+		c := meta.Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			mf := &tsdb.MeasurementFields{}
+			if err := mf.UnmarshalBinary(v); err != nil {
+				return err
+			}
+			r.fields[string(k)] = mf
+			r.codecs[string(k)] = tsdb.NewFieldCodec(mf.Fields)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	seriesSet := make(map[string]bool)
+
+	// ignore series index and find all series in this shard
+	if err := r.db.View(func(tx *bolt.Tx) error {
+		tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
+			key := string(name)
+			if !excludedBuckets[key] {
+				seriesSet[key] = true
+			}
+			return nil
+		})
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	r.tx, err = r.db.Begin(false)
+	if err != nil {
+		return err
+	}
+
+	// Create cursor for each field of each series.
+	for s := range seriesSet {
+		measurement := tsdb.MeasurementFromSeriesKey(s)
+		fields := r.fields[measurement]
+		if fields == nil {
+			r.stats.IncrFiltered()
+			continue
+		}
+		for _, f := range fields.Fields {
+			c := newCursor(r.tx, s, f.Name, r.codecs[measurement])
+			c.SeekTo(0)
+			r.cursors = append(r.cursors, c)
+		}
+	}
+	sort.Sort(cursors(r.cursors))
+
+	return nil
+}
+
+// Next returns whether any data remains to be read. It must be called before
+// the next call to Read().
+func (r *Reader) Next() bool {
+	r.valuePos = 0
+OUTER:
+	for {
+		if r.currCursor >= len(r.cursors) {
+			// All cursors drained. No more data remains.
+			return false
+		}
+
+		cc := r.cursors[r.currCursor]
+		r.keyBuf = tsm1.SeriesFieldKey(cc.series, cc.field)
+
+		for {
+			k, v := cc.Next()
+			if k == -1 {
+				// Go to next cursor and try again.
+				r.currCursor++
+				if r.valuePos == 0 {
+					// The previous cursor had no data. Instead of returning
+					// just go immediately to the next cursor.
+					continue OUTER
+				}
+				// There is some data available. Indicate that it should be read.
+				return true
+			}
+
+			if f, ok := v.(float64); ok {
+				if math.IsInf(f, 0) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrInf()
+					continue
+				}
+
+				if math.IsNaN(f) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrNaN()
+					continue
+				}
+			}
+
+			r.values[r.valuePos] = tsm1.NewValue(k, v)
+			r.valuePos++
+
+			if r.valuePos >= len(r.values) {
+				return true
+			}
+		}
+	}
+}
+
+// Read returns the next chunk of data in the shard, converted to tsm1 values. Data is
+// emitted completely for every field, in every series, before the next field is processed.
+// Data from Read() adheres to the requirements for writing to tsm1 shards
+func (r *Reader) Read() (string, []tsm1.Value, error) {
+	return r.keyBuf, r.values[:r.valuePos], nil
+}
+
+// Close closes the reader.
+func (r *Reader) Close() error {
+	r.tx.Rollback()
+	return r.db.Close()
+}
+
+// cursor provides ordered iteration across a series.
+type cursor struct {
+	// Bolt cursor and readahead buffer.
+	cursor *bolt.Cursor
+	keyBuf int64
+	valBuf interface{}
+
+	series string
+	field  string
+	dec    *tsdb.FieldCodec
+}
+
+// Cursor returns an iterator for a key over a single field.
+func newCursor(tx *bolt.Tx, series string, field string, dec *tsdb.FieldCodec) *cursor {
+	cur := &cursor{
+		keyBuf: -2,
+		series: series,
+		field:  field,
+		dec:    dec,
+	}
+
+	// Retrieve series bucket.
+	b := tx.Bucket([]byte(series))
+	if b != nil {
+		cur.cursor = b.Cursor()
+	}
+
+	return cur
+}
+
+// Seek moves the cursor to a position.
+func (c *cursor) SeekTo(seek int64) {
+	var seekBytes [8]byte
+	binary.BigEndian.PutUint64(seekBytes[:], uint64(seek))
+	k, v := c.cursor.Seek(seekBytes[:])
+	c.keyBuf, c.valBuf = tsdb.DecodeKeyValue(c.field, c.dec, k, v)
+}
+
+// Next returns the next key/value pair from the cursor.
+func (c *cursor) Next() (key int64, value interface{}) {
+	for {
+		k, v := func() (int64, interface{}) {
+			if c.keyBuf != -2 {
+				k, v := c.keyBuf, c.valBuf
+				c.keyBuf = -2
+				return k, v
+			}
+
+			k, v := c.cursor.Next()
+			if k == nil {
+				return -1, nil
+			}
+			return tsdb.DecodeKeyValue(c.field, c.dec, k, v)
+		}()
+
+		if k != -1 && v == nil {
+			// There is a point in the series at the next timestamp,
+			// but not for this cursor's field. Go to the next point.
+			continue
+		}
+		return k, v
+	}
+}
+
+// Sort b1 cursors in correct order for writing to TSM files.
+
+type cursors []*cursor
+
+func (a cursors) Len() int      { return len(a) }
+func (a cursors) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a cursors) Less(i, j int) bool {
+	if a[i].series == a[j].series {
+		return a[i].field < a[j].field
+	}
+	return a[i].series < a[j].series
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/bz1/reader.go

@@ -0,0 +1,371 @@
+// Package bz1 reads data from bz1 shards.
+package bz1 // import "github.com/influxdata/influxdb/cmd/influx_tsm/bz1"
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/golang/snappy"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// DefaultChunkSize is the size of chunks read from the bz1 shard
+const DefaultChunkSize = 1000
+
+// Reader is used to read all data from a bz1 shard.
+type Reader struct {
+	path string
+	db   *bolt.DB
+	tx   *bolt.Tx
+
+	cursors    []*cursor
+	currCursor int
+
+	keyBuf   string
+	values   []tsm1.Value
+	valuePos int
+
+	fields map[string]*tsdb.MeasurementFields
+	codecs map[string]*tsdb.FieldCodec
+
+	stats *stats.Stats
+}
+
+// NewReader returns a reader for the bz1 shard at path.
+func NewReader(path string, stats *stats.Stats, chunkSize int) *Reader {
+	r := &Reader{
+		path:   path,
+		fields: make(map[string]*tsdb.MeasurementFields),
+		codecs: make(map[string]*tsdb.FieldCodec),
+		stats:  stats,
+	}
+
+	if chunkSize <= 0 {
+		chunkSize = DefaultChunkSize
+	}
+
+	r.values = make([]tsm1.Value, chunkSize)
+
+	return r
+}
+
+// Open opens the reader.
+func (r *Reader) Open() error {
+	// Open underlying storage.
+	db, err := bolt.Open(r.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	r.db = db
+
+	seriesSet := make(map[string]bool)
+
+	if err := r.db.View(func(tx *bolt.Tx) error {
+		var data []byte
+
+		meta := tx.Bucket([]byte("meta"))
+		if meta == nil {
+			// No data in this shard.
+			return nil
+		}
+
+		pointsBucket := tx.Bucket([]byte("points"))
+		if pointsBucket == nil {
+			return nil
+		}
+
+		if err := pointsBucket.ForEach(func(key, _ []byte) error {
+			seriesSet[string(key)] = true
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		buf := meta.Get([]byte("fields"))
+		if buf == nil {
+			// No data in this shard.
+			return nil
+		}
+
+		data, err = snappy.Decode(nil, buf)
+		if err != nil {
+			return err
+		}
+		if err := json.Unmarshal(data, &r.fields); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// Build the codec for each measurement.
+	for k, v := range r.fields {
+		r.codecs[k] = tsdb.NewFieldCodec(v.Fields)
+	}
+
+	r.tx, err = r.db.Begin(false)
+	if err != nil {
+		return err
+	}
+
+	// Create cursor for each field of each series.
+	for s := range seriesSet {
+		measurement := tsdb.MeasurementFromSeriesKey(s)
+		fields := r.fields[measurement]
+		if fields == nil {
+			r.stats.IncrFiltered()
+			continue
+		}
+		for _, f := range fields.Fields {
+			c := newCursor(r.tx, s, f.Name, r.codecs[measurement])
+			if c == nil {
+				continue
+			}
+			c.SeekTo(0)
+			r.cursors = append(r.cursors, c)
+		}
+	}
+	sort.Sort(cursors(r.cursors))
+
+	return nil
+}
+
+// Next returns whether there is any more data to be read.
+func (r *Reader) Next() bool {
+	r.valuePos = 0
+OUTER:
+	for {
+		if r.currCursor >= len(r.cursors) {
+			// All cursors drained. No more data remains.
+			return false
+		}
+
+		cc := r.cursors[r.currCursor]
+		r.keyBuf = tsm1.SeriesFieldKey(cc.series, cc.field)
+
+		for {
+			k, v := cc.Next()
+			if k == -1 {
+				// Go to next cursor and try again.
+				r.currCursor++
+				if r.valuePos == 0 {
+					// The previous cursor had no data. Instead of returning
+					// just go immediately to the next cursor.
+					continue OUTER
+				}
+				// There is some data available. Indicate that it should be read.
+				return true
+			}
+
+			if f, ok := v.(float64); ok {
+				if math.IsInf(f, 0) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrInf()
+					continue
+				}
+
+				if math.IsNaN(f) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrNaN()
+					continue
+				}
+			}
+
+			r.values[r.valuePos] = tsm1.NewValue(k, v)
+			r.valuePos++
+
+			if r.valuePos >= len(r.values) {
+				return true
+			}
+		}
+	}
+}
+
+// Read returns the next chunk of data in the shard, converted to tsm1 values. Data is
+// emitted completely for every field, in every series, before the next field is processed.
+// Data from Read() adheres to the requirements for writing to tsm1 shards
+func (r *Reader) Read() (string, []tsm1.Value, error) {
+	return r.keyBuf, r.values[:r.valuePos], nil
+}
+
+// Close closes the reader.
+func (r *Reader) Close() error {
+	r.tx.Rollback()
+	return r.db.Close()
+}
+
+// cursor provides ordered iteration across a series.
+type cursor struct {
+	cursor       *bolt.Cursor
+	buf          []byte // uncompressed buffer
+	off          int    // buffer offset
+	fieldIndices []int
+	index        int
+
+	series string
+	field  string
+	dec    *tsdb.FieldCodec
+
+	keyBuf int64
+	valBuf interface{}
+}
+
+// newCursor returns an instance of a bz1 cursor.
+func newCursor(tx *bolt.Tx, series string, field string, dec *tsdb.FieldCodec) *cursor {
+	// Retrieve points bucket. Ignore if there is no bucket.
+	b := tx.Bucket([]byte("points")).Bucket([]byte(series))
+	if b == nil {
+		return nil
+	}
+
+	return &cursor{
+		cursor: b.Cursor(),
+		series: series,
+		field:  field,
+		dec:    dec,
+		keyBuf: -2,
+	}
+}
+
+// Seek moves the cursor to a position.
+func (c *cursor) SeekTo(seek int64) {
+	var seekBytes [8]byte
+	binary.BigEndian.PutUint64(seekBytes[:], uint64(seek))
+
+	// Move cursor to appropriate block and set to buffer.
+	k, v := c.cursor.Seek(seekBytes[:])
+	if v == nil { // get the last block, it might have this time
+		_, v = c.cursor.Last()
+	} else if seek < int64(binary.BigEndian.Uint64(k)) { // the seek key is less than this block, go back one and check
+		_, v = c.cursor.Prev()
+
+		// if the previous block max time is less than the seek value, reset to where we were originally
+		if v == nil || seek > int64(binary.BigEndian.Uint64(v[0:8])) {
+			_, v = c.cursor.Seek(seekBytes[:])
+		}
+	}
+	c.setBuf(v)
+
+	// Read current block up to seek position.
+	c.seekBuf(seekBytes[:])
+
+	// Return current entry.
+	c.keyBuf, c.valBuf = c.read()
+}
+
+// seekBuf moves the cursor to a position within the current buffer.
+func (c *cursor) seekBuf(seek []byte) (key, value []byte) {
+	for {
+		// Slice off the current entry.
+		buf := c.buf[c.off:]
+
+		// Exit if current entry's timestamp is on or after the seek.
+		if len(buf) == 0 {
+			return
+		}
+
+		if bytes.Compare(buf[0:8], seek) != -1 {
+			return
+		}
+
+		c.off += entryHeaderSize + entryDataSize(buf)
+	}
+}
+
+// Next returns the next key/value pair from the cursor. If there are no values
+// remaining, -1 is returned.
+func (c *cursor) Next() (int64, interface{}) {
+	for {
+		k, v := func() (int64, interface{}) {
+			if c.keyBuf != -2 {
+				k, v := c.keyBuf, c.valBuf
+				c.keyBuf = -2
+				return k, v
+			}
+
+			// Ignore if there is no buffer.
+			if len(c.buf) == 0 {
+				return -1, nil
+			}
+
+			// Move forward to next entry.
+			c.off += entryHeaderSize + entryDataSize(c.buf[c.off:])
+
+			// If no items left then read first item from next block.
+			if c.off >= len(c.buf) {
+				_, v := c.cursor.Next()
+				c.setBuf(v)
+			}
+
+			return c.read()
+		}()
+
+		if k != -1 && v == nil {
+			// There is a point in the series at the next timestamp,
+			// but not for this cursor's field. Go to the next point.
+			continue
+		}
+		return k, v
+	}
+}
+
+// setBuf saves a compressed block to the buffer.
+func (c *cursor) setBuf(block []byte) {
+	// Clear if the block is empty.
+	if len(block) == 0 {
+		c.buf, c.off, c.fieldIndices, c.index = c.buf[0:0], 0, c.fieldIndices[0:0], 0
+		return
+	}
+
+	// Otherwise decode block into buffer.
+	// Skip over the first 8 bytes since they are the max timestamp.
+	buf, err := snappy.Decode(nil, block[8:])
+	if err != nil {
+		c.buf = c.buf[0:0]
+		fmt.Printf("block decode error: %s\n", err)
+	}
+
+	c.buf, c.off = buf, 0
+}
+
+// read reads the current key and value from the current block.
+func (c *cursor) read() (key int64, value interface{}) {
+	// Return nil if the offset is at the end of the buffer.
+	if c.off >= len(c.buf) {
+		return -1, nil
+	}
+
+	// Otherwise read the current entry.
+	buf := c.buf[c.off:]
+	dataSize := entryDataSize(buf)
+
+	return tsdb.DecodeKeyValue(c.field, c.dec, buf[0:8], buf[entryHeaderSize:entryHeaderSize+dataSize])
+}
+
+// Sort bz1 cursors in correct order for writing to TSM files.
+
+type cursors []*cursor
+
+func (a cursors) Len() int      { return len(a) }
+func (a cursors) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a cursors) Less(i, j int) bool {
+	if a[i].series == a[j].series {
+		return a[i].field < a[j].field
+	}
+	return a[i].series < a[j].series
+}
+
+// entryHeaderSize is the number of bytes required for the header.
+const entryHeaderSize = 8 + 4
+
+// entryDataSize returns the size of an entry's data field, in bytes.
+func entryDataSize(v []byte) int { return int(binary.BigEndian.Uint32(v[8:12])) }

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/converter.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+const (
+	maxBlocksPerKey = 65535
+)
+
+// KeyIterator is used to iterate over b* keys for conversion to tsm keys
+type KeyIterator interface {
+	Next() bool
+	Read() (string, []tsm1.Value, error)
+}
+
+// Converter encapsulates the logic for converting b*1 shards to tsm1 shards.
+type Converter struct {
+	path           string
+	maxTSMFileSize uint32
+	sequence       int
+	stats          *stats.Stats
+}
+
+// NewConverter returns a new instance of the Converter.
+func NewConverter(path string, sz uint32, stats *stats.Stats) *Converter {
+	return &Converter{
+		path:           path,
+		maxTSMFileSize: sz,
+		stats:          stats,
+	}
+}
+
+// Process writes the data provided by iter to a tsm1 shard.
+func (c *Converter) Process(iter KeyIterator) error {
+	// Ensure the tsm1 directory exists.
+	if err := os.MkdirAll(c.path, 0777); err != nil {
+		return err
+	}
+
+	// Iterate until no more data remains.
+	var w tsm1.TSMWriter
+	var keyCount map[string]int
+
+	for iter.Next() {
+		k, v, err := iter.Read()
+		if err != nil {
+			return err
+		}
+
+		if w == nil {
+			w, err = c.nextTSMWriter()
+			if err != nil {
+				return err
+			}
+			keyCount = map[string]int{}
+		}
+		if err := w.Write(k, v); err != nil {
+			return err
+		}
+		keyCount[k]++
+
+		c.stats.AddPointsRead(len(v))
+		c.stats.AddPointsWritten(len(v))
+
+		// If we have a max file size configured and we're over it, start a new TSM file.
+		if w.Size() > c.maxTSMFileSize || keyCount[k] == maxBlocksPerKey {
+			if err := w.WriteIndex(); err != nil && err != tsm1.ErrNoValues {
+				return err
+			}
+
+			c.stats.AddTSMBytes(w.Size())
+
+			if err := w.Close(); err != nil {
+				return err
+			}
+			w = nil
+		}
+	}
+
+	if w != nil {
+		if err := w.WriteIndex(); err != nil && err != tsm1.ErrNoValues {
+			return err
+		}
+		c.stats.AddTSMBytes(w.Size())
+
+		if err := w.Close(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// nextTSMWriter returns the next TSMWriter for the Converter.
+func (c *Converter) nextTSMWriter() (tsm1.TSMWriter, error) {
+	c.sequence++
+	fileName := filepath.Join(c.path, fmt.Sprintf("%09d-%09d.%s", 1, c.sequence, tsm1.TSMFileExtension))
+
+	fd, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the writer for the new TSM file.
+	w, err := tsm1.NewTSMWriter(fd)
+	if err != nil {
+		return nil, err
+	}
+
+	c.stats.IncrTSMFileCount()
+	return w, nil
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/main.go

@@ -0,0 +1,415 @@
+// Command influx_tsm converts b1 or bz1 shards (from InfluxDB releases earlier than v0.11)
+// to the current tsm1 format.
+package main
+
+import (
+	"bufio"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/b1"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/bz1"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+)
+
+// ShardReader reads b* shards and converts to tsm shards
+type ShardReader interface {
+	KeyIterator
+	Open() error
+	Close() error
+}
+
+const (
+	tsmExt = "tsm"
+)
+
+var description = `
+Convert a database from b1 or bz1 format to tsm1 format.
+
+This tool will backup the directories before conversion (if not disabled).
+The backed-up files must be removed manually, generally after starting up the
+node again to make sure all of data has been converted correctly.
+
+To restore a backup:
+  Shut down the node, remove the converted directory, and
+  copy the backed-up directory to the original location.`
+
+type options struct {
+	DataPath       string
+	BackupPath     string
+	DBs            []string
+	DebugAddr      string
+	TSMSize        uint64
+	Parallel       bool
+	SkipBackup     bool
+	UpdateInterval time.Duration
+	Yes            bool
+	CPUFile        string
+}
+
+func (o *options) Parse() error {
+	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	var dbs string
+
+	fs.StringVar(&dbs, "dbs", "", "Comma-delimited list of databases to convert. Default is to convert all databases.")
+	fs.Uint64Var(&opts.TSMSize, "sz", maxTSMSz, "Maximum size of individual TSM files.")
+	fs.BoolVar(&opts.Parallel, "parallel", false, "Perform parallel conversion. (up to GOMAXPROCS shards at once)")
+	fs.BoolVar(&opts.SkipBackup, "nobackup", false, "Disable database backups. Not recommended.")
+	fs.StringVar(&opts.BackupPath, "backup", "", "The location to backup up the current databases. Must not be within the data directory.")
+	fs.StringVar(&opts.DebugAddr, "debug", "", "If set, http debugging endpoints will be enabled on the given address")
+	fs.DurationVar(&opts.UpdateInterval, "interval", 5*time.Second, "How often status updates are printed.")
+	fs.BoolVar(&opts.Yes, "y", false, "Don't ask, just convert")
+	fs.StringVar(&opts.CPUFile, "profile", "", "CPU Profile location")
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %v [options] <data-path> \n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "%v\n\nOptions:\n", description)
+		fs.PrintDefaults()
+		fmt.Fprintf(os.Stderr, "\n")
+	}
+
+	if err := fs.Parse(os.Args[1:]); err != nil {
+		return err
+	}
+
+	if len(fs.Args()) < 1 {
+		return errors.New("no data directory specified")
+	}
+	var err error
+	if o.DataPath, err = filepath.Abs(fs.Args()[0]); err != nil {
+		return err
+	}
+	if o.DataPath, err = filepath.EvalSymlinks(filepath.Clean(o.DataPath)); err != nil {
+		return err
+	}
+
+	if o.TSMSize > maxTSMSz {
+		return fmt.Errorf("bad TSM file size, maximum TSM file size is %d", maxTSMSz)
+	}
+
+	// Check if specific databases were requested.
+	o.DBs = strings.Split(dbs, ",")
+	if len(o.DBs) == 1 && o.DBs[0] == "" {
+		o.DBs = nil
+	}
+
+	if !o.SkipBackup {
+		if o.BackupPath == "" {
+			return errors.New("either -nobackup or -backup DIR must be set")
+		}
+		if o.BackupPath, err = filepath.Abs(o.BackupPath); err != nil {
+			return err
+		}
+		if o.BackupPath, err = filepath.EvalSymlinks(filepath.Clean(o.BackupPath)); err != nil {
+			if os.IsNotExist(err) {
+				return errors.New("backup directory must already exist")
+			}
+			return err
+		}
+
+		if strings.HasPrefix(o.BackupPath, o.DataPath) {
+			fmt.Println(o.BackupPath, o.DataPath)
+			return errors.New("backup directory cannot be contained within data directory")
+		}
+	}
+
+	if o.DebugAddr != "" {
+		log.Printf("Starting debugging server on http://%v", o.DebugAddr)
+		go func() {
+			log.Fatal(http.ListenAndServe(o.DebugAddr, nil))
+		}()
+	}
+
+	return nil
+}
+
+var opts options
+
+const maxTSMSz uint64 = 2 * 1024 * 1024 * 1024
+
+func init() {
+	log.SetOutput(os.Stderr)
+	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
+}
+
+func main() {
+	if err := opts.Parse(); err != nil {
+		log.Fatal(err)
+	}
+
+	// Determine the list of databases
+	dbs, err := ioutil.ReadDir(opts.DataPath)
+	if err != nil {
+		log.Fatalf("failed to access data directory at %v: %v\n", opts.DataPath, err)
+	}
+	fmt.Println() // Cleanly separate output from start of program.
+
+	if opts.Parallel {
+		if !isEnvSet("GOMAXPROCS") {
+			// Only modify GOMAXPROCS if it wasn't set in the environment
+			// This means 'GOMAXPROCS=1 influx_tsm -parallel' will not actually
+			// run in parallel
+			runtime.GOMAXPROCS(runtime.NumCPU())
+		}
+	}
+
+	var badUser string
+	if opts.SkipBackup {
+		badUser = "(NOT RECOMMENDED)"
+	}
+
+	// Dump summary of what is about to happen.
+	fmt.Println("b1 and bz1 shard conversion.")
+	fmt.Println("-----------------------------------")
+	fmt.Println("Data directory is:                 ", opts.DataPath)
+	if !opts.SkipBackup {
+		fmt.Println("Backup directory is:               ", opts.BackupPath)
+	}
+	fmt.Println("Databases specified:               ", allDBs(opts.DBs))
+	fmt.Println("Database backups enabled:          ", yesno(!opts.SkipBackup), badUser)
+	fmt.Printf("Parallel mode enabled (GOMAXPROCS): %s (%d)\n", yesno(opts.Parallel), runtime.GOMAXPROCS(0))
+	fmt.Println()
+
+	shards := collectShards(dbs)
+
+	// Anything to convert?
+	fmt.Printf("\nFound %d shards that will be converted.\n", len(shards))
+	if len(shards) == 0 {
+		fmt.Println("Nothing to do.")
+		return
+	}
+
+	// Display list of convertible shards.
+	fmt.Println()
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, 0, 8, 1, '\t', 0)
+	fmt.Fprintln(w, "Database\tRetention\tPath\tEngine\tSize")
+	for _, si := range shards {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%d\n", si.Database, si.RetentionPolicy, si.FullPath(opts.DataPath), si.FormatAsString(), si.Size)
+	}
+	w.Flush()
+
+	if !opts.Yes {
+		// Get confirmation from user.
+		fmt.Printf("\nThese shards will be converted. Proceed? y/N: ")
+		liner := bufio.NewReader(os.Stdin)
+		yn, err := liner.ReadString('\n')
+		if err != nil {
+			log.Fatalf("failed to read response: %v", err)
+		}
+		yn = strings.TrimRight(strings.ToLower(yn), "\n")
+		if yn != "y" {
+			log.Fatal("Conversion aborted.")
+		}
+	}
+	fmt.Println("Conversion starting....")
+
+	if opts.CPUFile != "" {
+		f, err := os.Create(opts.CPUFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if err = pprof.StartCPUProfile(f); err != nil {
+			log.Fatal(err)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
+	tr := newTracker(shards, opts)
+
+	if err := tr.Run(); err != nil {
+		log.Fatalf("Error occurred preventing completion: %v\n", err)
+	}
+
+	tr.PrintStats()
+}
+
+func collectShards(dbs []os.FileInfo) tsdb.ShardInfos {
+	// Get the list of shards for conversion.
+	var shards tsdb.ShardInfos
+	for _, db := range dbs {
+		d := tsdb.NewDatabase(filepath.Join(opts.DataPath, db.Name()))
+		shs, err := d.Shards()
+		if err != nil {
+			log.Fatalf("Failed to access shards for database %v: %v\n", d.Name(), err)
+		}
+		shards = append(shards, shs...)
+	}
+
+	sort.Sort(shards)
+	shards = shards.FilterFormat(tsdb.TSM1)
+	if len(dbs) > 0 {
+		shards = shards.ExclusiveDatabases(opts.DBs)
+	}
+
+	return shards
+}
+
+// backupDatabase backs up the database named db
+func backupDatabase(db string) error {
+	copyFile := func(path string, info os.FileInfo, err error) error {
+		// Strip the DataPath from the path and replace with BackupPath.
+		toPath := strings.Replace(path, opts.DataPath, opts.BackupPath, 1)
+
+		if info.IsDir() {
+			return os.MkdirAll(toPath, info.Mode())
+		}
+
+		in, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer in.Close()
+
+		srcInfo, err := os.Stat(path)
+		if err != nil {
+			return err
+		}
+
+		out, err := os.OpenFile(toPath, os.O_CREATE|os.O_WRONLY, info.Mode())
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+
+		dstInfo, err := os.Stat(toPath)
+		if err != nil {
+			return err
+		}
+
+		if dstInfo.Size() == srcInfo.Size() {
+			log.Printf("Backup file already found for %v with correct size, skipping.", path)
+			return nil
+		}
+
+		if dstInfo.Size() > srcInfo.Size() {
+			log.Printf("Invalid backup file found for %v, replacing with good copy.", path)
+			if err := out.Truncate(0); err != nil {
+				return err
+			}
+			if _, err := out.Seek(0, io.SeekStart); err != nil {
+				return err
+			}
+		}
+
+		if dstInfo.Size() > 0 {
+			log.Printf("Resuming backup of file %v, starting at %v bytes", path, dstInfo.Size())
+		}
+
+		off, err := out.Seek(0, io.SeekEnd)
+		if err != nil {
+			return err
+		}
+		if _, err := in.Seek(off, io.SeekStart); err != nil {
+			return err
+		}
+
+		log.Printf("Backing up file %v", path)
+
+		_, err = io.Copy(out, in)
+
+		return err
+	}
+
+	return filepath.Walk(filepath.Join(opts.DataPath, db), copyFile)
+}
+
+// convertShard converts the shard in-place.
+func convertShard(si *tsdb.ShardInfo, tr *tracker) error {
+	src := si.FullPath(opts.DataPath)
+	dst := fmt.Sprintf("%v.%v", src, tsmExt)
+
+	var reader ShardReader
+	switch si.Format {
+	case tsdb.BZ1:
+		reader = bz1.NewReader(src, &tr.Stats, 0)
+	case tsdb.B1:
+		reader = b1.NewReader(src, &tr.Stats, 0)
+	default:
+		return fmt.Errorf("Unsupported shard format: %v", si.FormatAsString())
+	}
+
+	// Open the shard, and create a converter.
+	if err := reader.Open(); err != nil {
+		return fmt.Errorf("Failed to open %v for conversion: %v", src, err)
+	}
+	defer reader.Close()
+	converter := NewConverter(dst, uint32(opts.TSMSize), &tr.Stats)
+
+	// Perform the conversion.
+	if err := converter.Process(reader); err != nil {
+		return fmt.Errorf("Conversion of %v failed: %v", src, err)
+	}
+
+	// Delete source shard, and rename new tsm1 shard.
+	if err := reader.Close(); err != nil {
+		return fmt.Errorf("Conversion of %v failed due to close: %v", src, err)
+	}
+
+	if err := os.RemoveAll(si.FullPath(opts.DataPath)); err != nil {
+		return fmt.Errorf("Deletion of %v failed: %v", src, err)
+	}
+	if err := os.Rename(dst, src); err != nil {
+		return fmt.Errorf("Rename of %v to %v failed: %v", dst, src, err)
+	}
+
+	return nil
+}
+
+// ParallelGroup allows the maximum parrallelism of a set of operations to be controlled.
+type ParallelGroup chan struct{}
+
+// NewParallelGroup returns a group which allows n operations to run in parallel. A value of 0
+// means no operations will ever run.
+func NewParallelGroup(n int) ParallelGroup {
+	return make(chan struct{}, n)
+}
+
+// Do executes one operation of the ParallelGroup
+func (p ParallelGroup) Do(f func()) {
+	p <- struct{}{} // acquire working slot
+	defer func() { <-p }()
+
+	f()
+}
+
+// yesno returns "yes" for true, "no" for false.
+func yesno(b bool) string {
+	if b {
+		return "yes"
+	}
+	return "no"
+}
+
+// allDBs returns "all" if all databases are requested for conversion.
+func allDBs(dbs []string) string {
+	if dbs == nil {
+		return "all"
+	}
+	return fmt.Sprintf("%v", dbs)
+}
+
+// isEnvSet checks to see if a variable was set in the environment
+func isEnvSet(name string) bool {
+	for _, s := range os.Environ() {
+		if strings.SplitN(s, "=", 2)[0] == name {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/stats/stats.go

@@ -0,0 +1,55 @@
+// Package stats contains statistics for converting non-TSM shards to TSM.
+package stats
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+// Stats are the statistics captured while converting non-TSM shards to TSM
+type Stats struct {
+	NanFiltered     uint64
+	InfFiltered     uint64
+	FieldsFiltered  uint64
+	PointsWritten   uint64
+	PointsRead      uint64
+	TsmFilesCreated uint64
+	TsmBytesWritten uint64
+	CompletedShards uint64
+	TotalTime       time.Duration
+}
+
+// AddPointsRead increments the number of read points.
+func (s *Stats) AddPointsRead(n int) {
+	atomic.AddUint64(&s.PointsRead, uint64(n))
+}
+
+// AddPointsWritten increments the number of written points.
+func (s *Stats) AddPointsWritten(n int) {
+	atomic.AddUint64(&s.PointsWritten, uint64(n))
+}
+
+// AddTSMBytes increments the number of TSM Bytes.
+func (s *Stats) AddTSMBytes(n uint32) {
+	atomic.AddUint64(&s.TsmBytesWritten, uint64(n))
+}
+
+// IncrTSMFileCount increments the number of TSM files created.
+func (s *Stats) IncrTSMFileCount() {
+	atomic.AddUint64(&s.TsmFilesCreated, 1)
+}
+
+// IncrNaN increments the number of NaNs filtered.
+func (s *Stats) IncrNaN() {
+	atomic.AddUint64(&s.NanFiltered, 1)
+}
+
+// IncrInf increments the number of Infs filtered.
+func (s *Stats) IncrInf() {
+	atomic.AddUint64(&s.InfFiltered, 1)
+}
+
+// IncrFiltered increments the number of fields filtered.
+func (s *Stats) IncrFiltered() {
+	atomic.AddUint64(&s.FieldsFiltered, 1)
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tracker.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+)
+
+// tracker will orchestrate and track the conversions of non-TSM shards to TSM
+type tracker struct {
+	Stats stats.Stats
+
+	shards tsdb.ShardInfos
+	opts   options
+
+	pg ParallelGroup
+	wg sync.WaitGroup
+}
+
+// newTracker will setup and return a clean tracker instance
+func newTracker(shards tsdb.ShardInfos, opts options) *tracker {
+	t := &tracker{
+		shards: shards,
+		opts:   opts,
+		pg:     NewParallelGroup(runtime.GOMAXPROCS(0)),
+	}
+
+	return t
+}
+
+func (t *tracker) Run() error {
+	conversionStart := time.Now()
+
+	// Backup each directory.
+	if !opts.SkipBackup {
+		databases := t.shards.Databases()
+		fmt.Printf("Backing up %d databases...\n", len(databases))
+		t.wg.Add(len(databases))
+		for i := range databases {
+			db := databases[i]
+			go t.pg.Do(func() {
+				defer t.wg.Done()
+
+				start := time.Now()
+				log.Printf("Backup of database '%v' started", db)
+				err := backupDatabase(db)
+				if err != nil {
+					log.Fatalf("Backup of database %v failed: %v\n", db, err)
+				}
+				log.Printf("Database %v backed up (%v)\n", db, time.Since(start))
+			})
+		}
+		t.wg.Wait()
+	} else {
+		fmt.Println("Database backup disabled.")
+	}
+
+	t.wg.Add(len(t.shards))
+	for i := range t.shards {
+		si := t.shards[i]
+		go t.pg.Do(func() {
+			defer func() {
+				atomic.AddUint64(&t.Stats.CompletedShards, 1)
+				t.wg.Done()
+			}()
+
+			start := time.Now()
+			log.Printf("Starting conversion of shard: %v", si.FullPath(opts.DataPath))
+			if err := convertShard(si, t); err != nil {
+				log.Fatalf("Failed to convert %v: %v\n", si.FullPath(opts.DataPath), err)
+			}
+			log.Printf("Conversion of %v successful (%v)\n", si.FullPath(opts.DataPath), time.Since(start))
+		})
+	}
+
+	done := make(chan struct{})
+	go func() {
+		t.wg.Wait()
+		close(done)
+	}()
+
+WAIT_LOOP:
+	for {
+		select {
+		case <-done:
+			break WAIT_LOOP
+		case <-time.After(opts.UpdateInterval):
+			t.StatusUpdate()
+		}
+	}
+
+	t.Stats.TotalTime = time.Since(conversionStart)
+
+	return nil
+}
+
+func (t *tracker) StatusUpdate() {
+	shardCount := atomic.LoadUint64(&t.Stats.CompletedShards)
+	pointCount := atomic.LoadUint64(&t.Stats.PointsRead)
+	pointWritten := atomic.LoadUint64(&t.Stats.PointsWritten)
+
+	log.Printf("Still Working: Completed Shards: %d/%d Points read/written: %d/%d", shardCount, len(t.shards), pointCount, pointWritten)
+}
+
+func (t *tracker) PrintStats() {
+	preSize := t.shards.Size()
+	postSize := int64(t.Stats.TsmBytesWritten)
+
+	fmt.Printf("\nSummary statistics\n========================================\n")
+	fmt.Printf("Databases converted:                 %d\n", len(t.shards.Databases()))
+	fmt.Printf("Shards converted:                    %d\n", len(t.shards))
+	fmt.Printf("TSM files created:                   %d\n", t.Stats.TsmFilesCreated)
+	fmt.Printf("Points read:                         %d\n", t.Stats.PointsRead)
+	fmt.Printf("Points written:                      %d\n", t.Stats.PointsWritten)
+	fmt.Printf("NaN filtered:                        %d\n", t.Stats.NanFiltered)
+	fmt.Printf("Inf filtered:                        %d\n", t.Stats.InfFiltered)
+	fmt.Printf("Points without fields filtered:      %d\n", t.Stats.FieldsFiltered)
+	fmt.Printf("Disk usage pre-conversion (bytes):   %d\n", preSize)
+	fmt.Printf("Disk usage post-conversion (bytes):  %d\n", postSize)
+	fmt.Printf("Reduction factor:                    %d%%\n", 100*(preSize-postSize)/preSize)
+	fmt.Printf("Bytes per TSM point:                 %.2f\n", float64(postSize)/float64(t.Stats.PointsWritten))
+	fmt.Printf("Total conversion time:               %v\n", t.Stats.TotalTime)
+	fmt.Println()
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/codec.go

@@ -0,0 +1,119 @@
+package tsdb
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math"
+)
+
+const (
+	fieldFloat   = 1
+	fieldInteger = 2
+	fieldBoolean = 3
+	fieldString  = 4
+)
+
+var (
+	// ErrFieldNotFound is returned when a field cannot be found.
+	ErrFieldNotFound = errors.New("field not found")
+
+	// ErrFieldUnmappedID is returned when the system is presented, during decode, with a field ID
+	// there is no mapping for.
+	ErrFieldUnmappedID = errors.New("field ID not mapped")
+)
+
+// FieldCodec provides encoding and decoding functionality for the fields of a given
+// Measurement.
+type FieldCodec struct {
+	fieldsByID   map[uint8]*Field
+	fieldsByName map[string]*Field
+}
+
+// NewFieldCodec returns a FieldCodec for the given Measurement. Must be called with
+// a RLock that protects the Measurement.
+func NewFieldCodec(fields map[string]*Field) *FieldCodec {
+	fieldsByID := make(map[uint8]*Field, len(fields))
+	fieldsByName := make(map[string]*Field, len(fields))
+	for _, f := range fields {
+		fieldsByID[f.ID] = f
+		fieldsByName[f.Name] = f
+	}
+	return &FieldCodec{fieldsByID: fieldsByID, fieldsByName: fieldsByName}
+}
+
+// FieldIDByName returns the ID for the given field.
+func (f *FieldCodec) FieldIDByName(s string) (uint8, error) {
+	fi := f.fieldsByName[s]
+	if fi == nil {
+		return 0, ErrFieldNotFound
+	}
+	return fi.ID, nil
+}
+
+// DecodeByID scans a byte slice for a field with the given ID, converts it to its
+// expected type, and return that value.
+func (f *FieldCodec) DecodeByID(targetID uint8, b []byte) (interface{}, error) {
+	var value interface{}
+	for {
+		if len(b) == 0 {
+			// No more bytes.
+			return nil, ErrFieldNotFound
+		}
+
+		field := f.fieldsByID[b[0]]
+		if field == nil {
+			// This can happen, though is very unlikely. If this node receives encoded data, to be written
+			// to disk, and is queried for that data before its metastore is updated, there will be no field
+			// mapping for the data during decode. All this can happen because data is encoded by the node
+			// that first received the write request, not the node that actually writes the data to disk.
+			// So if this happens, the read must be aborted.
+			return nil, ErrFieldUnmappedID
+		}
+
+		switch field.Type {
+		case fieldFloat:
+			if field.ID == targetID {
+				value = math.Float64frombits(binary.BigEndian.Uint64(b[1:9]))
+			}
+			b = b[9:]
+		case fieldInteger:
+			if field.ID == targetID {
+				value = int64(binary.BigEndian.Uint64(b[1:9]))
+			}
+			b = b[9:]
+		case fieldBoolean:
+			if field.ID == targetID {
+				value = b[1] == 1
+			}
+			b = b[2:]
+		case fieldString:
+			length := binary.BigEndian.Uint16(b[1:3])
+			if field.ID == targetID {
+				value = string(b[3 : 3+length])
+			}
+			b = b[3+length:]
+		default:
+			panic(fmt.Sprintf("unsupported value type during decode by id: %T", field.Type))
+		}
+
+		if value != nil {
+			return value, nil
+		}
+	}
+}
+
+// DecodeByName scans a byte slice for a field with the given name, converts it to its
+// expected type, and return that value.
+func (f *FieldCodec) DecodeByName(name string, b []byte) (interface{}, error) {
+	fi := f.FieldByName(name)
+	if fi == nil {
+		return 0, ErrFieldNotFound
+	}
+	return f.DecodeByID(fi.ID, b)
+}
+
+// FieldByName returns the field by its name. It will return a nil if not found
+func (f *FieldCodec) FieldByName(name string) *Field {
+	return f.fieldsByName[name]
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/database.go

@@ -0,0 +1,244 @@
+// Pacage tsdb abstracts the various shard types supported by the influx_tsm command.
+package tsdb // import "github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/influxdata/influxdb/pkg/slices"
+)
+
+// Flags for differentiating between engines
+const (
+	B1 = iota
+	BZ1
+	TSM1
+)
+
+// EngineFormat holds the flag for the engine
+type EngineFormat int
+
+// String returns the string format of the engine.
+func (e EngineFormat) String() string {
+	switch e {
+	case TSM1:
+		return "tsm1"
+	case B1:
+		return "b1"
+	case BZ1:
+		return "bz1"
+	default:
+		panic("unrecognized shard engine format")
+	}
+}
+
+// ShardInfo is the description of a shard on disk.
+type ShardInfo struct {
+	Database        string
+	RetentionPolicy string
+	Path            string
+	Format          EngineFormat
+	Size            int64
+}
+
+// FormatAsString returns the format of the shard as a string.
+func (s *ShardInfo) FormatAsString() string {
+	return s.Format.String()
+}
+
+// FullPath returns the full path to the shard, given the data directory root.
+func (s *ShardInfo) FullPath(dataPath string) string {
+	return filepath.Join(dataPath, s.Database, s.RetentionPolicy, s.Path)
+}
+
+// ShardInfos is an array of ShardInfo
+type ShardInfos []*ShardInfo
+
+func (s ShardInfos) Len() int      { return len(s) }
+func (s ShardInfos) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s ShardInfos) Less(i, j int) bool {
+	if s[i].Database == s[j].Database {
+		if s[i].RetentionPolicy == s[j].RetentionPolicy {
+			return s[i].Path < s[j].Path
+		}
+
+		return s[i].RetentionPolicy < s[j].RetentionPolicy
+	}
+
+	return s[i].Database < s[j].Database
+}
+
+// Databases returns the sorted unique set of databases for the shards.
+func (s ShardInfos) Databases() []string {
+	dbm := make(map[string]bool)
+	for _, ss := range s {
+		dbm[ss.Database] = true
+	}
+
+	var dbs []string
+	for k := range dbm {
+		dbs = append(dbs, k)
+	}
+	sort.Strings(dbs)
+	return dbs
+}
+
+// FilterFormat returns a copy of the ShardInfos, with shards of the given
+// format removed.
+func (s ShardInfos) FilterFormat(fmt EngineFormat) ShardInfos {
+	var a ShardInfos
+	for _, si := range s {
+		if si.Format != fmt {
+			a = append(a, si)
+		}
+	}
+	return a
+}
+
+// Size returns the space on disk consumed by the shards.
+func (s ShardInfos) Size() int64 {
+	var sz int64
+	for _, si := range s {
+		sz += si.Size
+	}
+	return sz
+}
+
+// ExclusiveDatabases returns a copy of the ShardInfo, with shards associated
+// with the given databases present. If the given set is empty, all databases
+// are returned.
+func (s ShardInfos) ExclusiveDatabases(exc []string) ShardInfos {
+	var a ShardInfos
+
+	// Empty set? Return everything.
+	if len(exc) == 0 {
+		a = make(ShardInfos, len(s))
+		copy(a, s)
+		return a
+	}
+
+	for _, si := range s {
+		if slices.Exists(exc, si.Database) {
+			a = append(a, si)
+		}
+	}
+	return a
+}
+
+// Database represents an entire database on disk.
+type Database struct {
+	path string
+}
+
+// NewDatabase creates a database instance using data at path.
+func NewDatabase(path string) *Database {
+	return &Database{path: path}
+}
+
+// Name returns the name of the database.
+func (d *Database) Name() string {
+	return path.Base(d.path)
+}
+
+// Path returns the path to the database.
+func (d *Database) Path() string {
+	return d.path
+}
+
+// Shards returns information for every shard in the database.
+func (d *Database) Shards() ([]*ShardInfo, error) {
+	fd, err := os.Open(d.path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get each retention policy.
+	rps, err := fd.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Process each retention policy.
+	var shardInfos []*ShardInfo
+	for _, rp := range rps {
+		rpfd, err := os.Open(filepath.Join(d.path, rp))
+		if err != nil {
+			return nil, err
+		}
+
+		// Process each shard
+		shards, err := rpfd.Readdirnames(-1)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, sh := range shards {
+			fmt, sz, err := shardFormat(filepath.Join(d.path, rp, sh))
+			if err != nil {
+				return nil, err
+			}
+
+			si := &ShardInfo{
+				Database:        d.Name(),
+				RetentionPolicy: path.Base(rp),
+				Path:            sh,
+				Format:          fmt,
+				Size:            sz,
+			}
+			shardInfos = append(shardInfos, si)
+		}
+	}
+
+	sort.Sort(ShardInfos(shardInfos))
+	return shardInfos, nil
+}
+
+// shardFormat returns the format and size on disk of the shard at path.
+func shardFormat(path string) (EngineFormat, int64, error) {
+	// If it's a directory then it's a tsm1 engine
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	if fi.Mode().IsDir() {
+		return TSM1, fi.Size(), nil
+	}
+
+	// It must be a BoltDB-based engine.
+	db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return 0, 0, err
+	}
+	defer db.Close()
+
+	var format EngineFormat
+	err = db.View(func(tx *bolt.Tx) error {
+		// Retrieve the meta bucket.
+		b := tx.Bucket([]byte("meta"))
+
+		// If no format is specified then it must be an original b1 database.
+		if b == nil {
+			format = B1
+			return nil
+		}
+
+		// There is an actual format indicator.
+		switch f := string(b.Get([]byte("format"))); f {
+		case "b1", "v1":
+			format = B1
+		case "bz1":
+			format = BZ1
+		default:
+			return fmt.Errorf("unrecognized engine format: %s", f)
+		}
+
+		return nil
+	})
+
+	return format, fi.Size(), err
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/internal/meta.pb.go

@@ -0,0 +1,122 @@
+// Code generated by protoc-gen-gogo.
+// source: internal/meta.proto
+// DO NOT EDIT!
+
+/*
+Package internal is a generated protocol buffer package.
+
+It is generated from these files:
+	internal/meta.proto
+
+It has these top-level messages:
+	Series
+	Tag
+	MeasurementFields
+	Field
+*/
+package internal
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+type Series struct {
+	Key              *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"`
+	Tags             []*Tag  `protobuf:"bytes,2,rep,name=Tags" json:"Tags,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Series) Reset()         { *m = Series{} }
+func (m *Series) String() string { return proto.CompactTextString(m) }
+func (*Series) ProtoMessage()    {}
+
+func (m *Series) GetKey() string {
+	if m != nil && m.Key != nil {
+		return *m.Key
+	}
+	return ""
+}
+
+func (m *Series) GetTags() []*Tag {
+	if m != nil {
+		return m.Tags
+	}
+	return nil
+}
+
+type Tag struct {
+	Key              *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"`
+	Value            *string `protobuf:"bytes,2,req,name=Value" json:"Value,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Tag) Reset()         { *m = Tag{} }
+func (m *Tag) String() string { return proto.CompactTextString(m) }
+func (*Tag) ProtoMessage()    {}
+
+func (m *Tag) GetKey() string {
+	if m != nil && m.Key != nil {
+		return *m.Key
+	}
+	return ""
+}
+
+func (m *Tag) GetValue() string {
+	if m != nil && m.Value != nil {
+		return *m.Value
+	}
+	return ""
+}
+
+type MeasurementFields struct {
+	Fields           []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *MeasurementFields) Reset()         { *m = MeasurementFields{} }
+func (m *MeasurementFields) String() string { return proto.CompactTextString(m) }
+func (*MeasurementFields) ProtoMessage()    {}
+
+func (m *MeasurementFields) GetFields() []*Field {
+	if m != nil {
+		return m.Fields
+	}
+	return nil
+}
+
+type Field struct {
+	ID               *int32  `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
+	Type             *int32  `protobuf:"varint,3,req,name=Type" json:"Type,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Field) Reset()         { *m = Field{} }
+func (m *Field) String() string { return proto.CompactTextString(m) }
+func (*Field) ProtoMessage()    {}
+
+func (m *Field) GetID() int32 {
+	if m != nil && m.ID != nil {
+		return *m.ID
+	}
+	return 0
+}
+
+func (m *Field) GetName() string {
+	if m != nil && m.Name != nil {
+		return *m.Name
+	}
+	return ""
+}
+
+func (m *Field) GetType() int32 {
+	if m != nil && m.Type != nil {
+		return *m.Type
+	}
+	return 0
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/types.go

@@ -0,0 +1,60 @@
+package tsdb
+
+import (
+	"encoding/binary"
+	"strings"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/internal"
+	"github.com/influxdata/influxdb/influxql"
+
+	"github.com/gogo/protobuf/proto"
+)
+
+// Field represents an encoded field.
+type Field struct {
+	ID   uint8             `json:"id,omitempty"`
+	Name string            `json:"name,omitempty"`
+	Type influxql.DataType `json:"type,omitempty"`
+}
+
+// MeasurementFields is a mapping from measurements to its fields.
+type MeasurementFields struct {
+	Fields map[string]*Field `json:"fields"`
+	Codec  *FieldCodec
+}
+
+// UnmarshalBinary decodes the object from a binary format.
+func (m *MeasurementFields) UnmarshalBinary(buf []byte) error {
+	var pb internal.MeasurementFields
+	if err := proto.Unmarshal(buf, &pb); err != nil {
+		return err
+	}
+	m.Fields = make(map[string]*Field)
+	for _, f := range pb.Fields {
+		m.Fields[f.GetName()] = &Field{ID: uint8(f.GetID()), Name: f.GetName(), Type: influxql.DataType(f.GetType())}
+	}
+	return nil
+}
+
+// Series represents a series in the shard.
+type Series struct {
+	Key  string
+	Tags map[string]string
+}
+
+// MeasurementFromSeriesKey returns the Measurement name for a given series.
+func MeasurementFromSeriesKey(key string) string {
+	return strings.SplitN(key, ",", 2)[0]
+}
+
+// DecodeKeyValue decodes the key and value from bytes.
+func DecodeKeyValue(field string, dec *FieldCodec, k, v []byte) (int64, interface{}) {
+	// Convert key to a timestamp.
+	key := int64(binary.BigEndian.Uint64(k[0:8]))
+
+	decValue, err := dec.DecodeByName(field, v)
+	if err != nil {
+		return key, nil
+	}
+	return key, decValue
+}