/* Timelinize Copyright (c) 2013 Matthew Holt This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ // Package kmlgx implements a data source for KML files with Google extensions (gx). package kmlgx import ( "context" "encoding/xml" "errors" "io/fs" "path" "strconv" "strings" "time" "github.com/timelinize/timelinize/datasources/googlelocation" "github.com/timelinize/timelinize/timeline" "go.uber.org/zap" "golang.org/x/net/html/charset" ) // This could be nearly identical to GPX importer except the XML structure is different. // I decided to not stream this file because the gx:Track entries can be in any order, // instead of each point clustered together. That's unlikely in practice but seems like // a bad assumption to make for the sake of performance on files that are rarely huge. // So we just swallow this thing whole. func init() { err := timeline.RegisterDataSource(timeline.DataSource{ Name: "kml", Title: "Keyhole (KML)", Icon: "kml.svg", Description: "A .kml file or folder of .kml files containing location tracks; requires Google extension namespace (gx)", NewOptions: func() any { return new(Options) }, NewFileImporter: func() timeline.FileImporter { return new(FileImporter) }, }) if err != nil { timeline.Log.Fatal("registering data source", zap.Error(err)) } } // Options configures the data source. type Options struct { // The ID of the owner entity. REQUIRED for linking entity in DB. // TODO: maybe an attribute ID instead, in case the data represents multiple people OwnerEntityID uint64 `json:"owner_entity_id"` // Options specific to the location processor. googlelocation.LocationProcessingOptions } // FileImporter implements the timeline.FileImporter interface. type FileImporter struct{} // Recognize returns whether the file or folder is supported. func (FileImporter) Recognize(_ context.Context, dirEntry timeline.DirEntry, _ timeline.RecognizeParams) (timeline.Recognition, error) { rec := timeline.Recognition{DirThreshold: .9} // we can import directories, but let the import planner figure that out; only recognize files if dirEntry.IsDir() { return rec, nil } // recognize by file extension, then verify by decoding the file // (TODO: If a recognize FastMode is created, opening and decoding the file should not happen in FastMode) if strings.ToLower(path.Ext(dirEntry.Name())) == ".kml" { file, err := dirEntry.Open(".") if err != nil { return rec, err } defer file.Close() dec := xml.NewDecoder(file) dec.CharsetReader = charset.NewReaderLabel // handle non-UTF-8 encodings var doc document err = dec.Decode(&doc) if err != nil { return rec, nil } if strings.HasPrefix(doc.XMLNSGX, "http://www.google.com/kml/ext/2") { rec.Confidence = 1.0 } } return rec, nil } // FileImport imports data from a folder or file. func (fi *FileImporter) FileImport(ctx context.Context, dirEntry timeline.DirEntry, params timeline.ImportParams) error { dsOpt := params.DataSourceOptions.(*Options) return fs.WalkDir(dirEntry.FS, dirEntry.Filename, func(fpath string, d fs.DirEntry, err error) error { if err != nil { return err } if err := ctx.Err(); err != nil { return err } if strings.HasPrefix(d.Name(), ".") { // skip hidden files & folders if d.IsDir() { return fs.SkipDir } return nil } if d.IsDir() { return nil // traverse into subdirectories } // skip unsupported file types if ext := strings.ToLower(path.Ext(d.Name())); ext != ".kml" { return nil } file, err := dirEntry.FS.Open(fpath) if err != nil { return err } defer file.Close() dec := xml.NewDecoder(file) dec.CharsetReader = charset.NewReaderLabel // handle non-UTF-8 encodings var doc document err = dec.Decode(&doc) if err != nil { return err } doc.i = -1 if doc.XMLNSGX != "http://www.google.com/kml/ext/2.2" { // actually, we may not need this version specifically; other versions might work just as well return errors.New("KML document does not support Google extension 'gx' namespace") } if len(doc.Document.Placemark.Track.When) != len(doc.Document.Placemark.Track.Coord) { return errors.New("corrupt gx:Track data in Placemark: number of timestamps does not match number of coordinates") } // create location processor to clean up any noisy raw data locProc, err := googlelocation.NewLocationProcessor(&doc, dsOpt.LocationProcessingOptions) if err != nil { return err } // iterate each resulting location point and process it as an item for { l, err := locProc.NextLocation(ctx) if err != nil { return err } if l == nil { break } item := &timeline.Item{ Classification: timeline.ClassLocation, Timestamp: l.Timestamp, Timespan: l.Timespan, Location: l.Location(), Owner: timeline.Entity{ ID: dsOpt.OwnerEntityID, }, Metadata: l.Metadata, } if params.Timeframe.ContainsItem(item, false) { params.Pipeline <- &timeline.Graph{Item: item} } } return nil }) } // NextLocation returns the next available point from the XML document. func (d *document) NextLocation(ctx context.Context) (*googlelocation.Location, error) { if err := ctx.Err(); err != nil { return nil, err } for { d.i++ if d.i >= len(d.Document.Placemark.Track.Coord) { return nil, nil } const requiredFields = 3 coordFields := strings.Fields(d.Document.Placemark.Track.Coord[d.i]) if len(coordFields) < requiredFields { continue } lon, lat, alt := coordFields[0], coordFields[1], coordFields[2] ts := d.Document.Placemark.Track.When[d.i] lonE7, err := googlelocation.FloatStringToIntE7(lon) if err != nil { continue } latE7, err := googlelocation.FloatStringToIntE7(lat) if err != nil { continue } altNum, _ := strconv.ParseFloat(alt, 64) // TODO: not really anything we can do about this error, but we should probably log it, yeah? return &googlelocation.Location{ Original: point{ when: ts, lat: lat, lon: lon, alt: alt, latE7: latE7, lonE7: lonE7, }, LatitudeE7: latE7, LongitudeE7: lonE7, Altitude: altNum, Timestamp: ts, }, nil } } type point struct { when time.Time lat, lon, alt string latE7, lonE7 int64 } type document struct { XMLName xml.Name `xml:"kml"` Text string `xml:",chardata"` XMLNS string `xml:"xmlns,attr"` XMLNSGX string `xml:"gx,attr"` XMLNSKML string `xml:"kml,attr"` XMLNSAtom string `xml:"atom,attr"` Document struct { Text string `xml:",chardata"` Name string `xml:"name"` Placemark struct { Text string `xml:",chardata"` Track struct { Text string `xml:",chardata"` When []time.Time `xml:"when"` Coord []string `xml:"coord"` } `xml:"Track"` } `xml:"Placemark"` } `xml:"Document"` i int // our iterator through the track points }