/* Timelinize Copyright (c) 2013 Matthew Holt This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ // Package gpx implements a data source for GPS Exchange Format (https://en.wikipedia.org/wiki/GPS_Exchange_Format). package gpx import ( "context" "encoding/xml" "errors" "fmt" "io" "io/fs" "path" "strings" "time" "github.com/timelinize/timelinize/datasources/googlelocation" "github.com/timelinize/timelinize/timeline" "go.uber.org/zap" "golang.org/x/net/html/charset" ) func init() { err := timeline.RegisterDataSource(timeline.DataSource{ Name: "gpx", Title: "GPS Exchange", Icon: "gpx.svg", Description: "A .gpx file or folder of .gpx files containing location tracks", NewOptions: func() any { return new(Options) }, NewFileImporter: func() timeline.FileImporter { return new(FileImporter) }, }) if err != nil { timeline.Log.Fatal("registering data source", zap.Error(err)) } } // Options configures the data source. type Options struct { // The ID of the owner entity. REQUIRED for linking entity in DB. // TODO: maybe an attribute ID instead, in case the data represents multiple people OwnerEntityID uint64 `json:"owner_entity_id"` // Options specific to the location processor. googlelocation.LocationProcessingOptions } // FileImporter implements the timeline.FileImporter interface. type FileImporter struct{} // Recognize returns whether the file is supported. func (FileImporter) Recognize(_ context.Context, dirEntry timeline.DirEntry, _ timeline.RecognizeParams) (timeline.Recognition, error) { rec := timeline.Recognition{DirThreshold: .9} // we can import directories, but let the import planner figure that out; only recognize files if dirEntry.IsDir() { return rec, nil } // recognize by file extension if strings.ToLower(path.Ext(dirEntry.Name())) == ".gpx" { rec.Confidence = 1 } return rec, nil } // FileImport imports data from a file or folder. func (fi *FileImporter) FileImport(ctx context.Context, dirEntry timeline.DirEntry, params timeline.ImportParams) error { dsOpt := params.DataSourceOptions.(*Options) owner := timeline.Entity{ ID: dsOpt.OwnerEntityID, } return fs.WalkDir(dirEntry.FS, dirEntry.Filename, func(fpath string, d fs.DirEntry, err error) error { if err != nil { return err } if err := ctx.Err(); err != nil { return err } if strings.HasPrefix(d.Name(), ".") { // skip hidden files & folders if d.IsDir() { return fs.SkipDir } return nil } if d.IsDir() { return nil // traverse into subdirectories } // skip unsupported file types if ext := strings.ToLower(path.Ext(d.Name())); ext != ".gpx" { return nil } file, err := dirEntry.FS.Open(fpath) if err != nil { return err } defer file.Close() proc, err := NewProcessor(file, owner, params, dsOpt.LocationProcessingOptions) if err != nil { return err } for { g, err := proc.NextGPXGraph(ctx) if err != nil { return err } if g == nil { break } params.Pipeline <- g } return nil }) } // Processor can get the next GPX point. Call NewProcessor to make a valid instance. type Processor struct { dec *decoder opt timeline.ImportParams locProc googlelocation.LocationSource owner timeline.Entity } // NewProcessor returns a new GPX processor. func NewProcessor(file io.Reader, owner timeline.Entity, opt timeline.ImportParams, locOpt googlelocation.LocationProcessingOptions) (*Processor, error) { // create XML decoder (wrapped to track some state as it decodes) xmlDec := &decoder{Decoder: xml.NewDecoder(file)} xmlDec.CharsetReader = charset.NewReaderLabel // handle non-UTF-8 encodings // create location processor to clean up any noisy raw data locProc, err := googlelocation.NewLocationProcessor(xmlDec, locOpt) if err != nil { return nil, err } return &Processor{ dec: xmlDec, opt: opt, locProc: locProc, owner: owner, }, nil } // NextGPXGraph returns the next GPX graph (item or entity) func (p *Processor) NextGPXGraph(ctx context.Context) (*timeline.Graph, error) { for { if err := ctx.Err(); err != nil { return nil, err } l, err := p.locProc.NextLocation(ctx) if err != nil { return nil, err } if l == nil { break } if l.Metadata == nil { l.Metadata = make(timeline.Metadata) } item := &timeline.Item{ Classification: timeline.ClassLocation, Timestamp: l.Timestamp, Timespan: l.Timespan, Location: l.Location(), Owner: p.owner, Metadata: l.Metadata, } g := &timeline.Graph{Item: item} makePlaceEntity := func(pt wpt) { // store named waypoints as place entities if pt.Name != "" { g.ToEntity(timeline.RelVisit, &timeline.Entity{ Type: timeline.EntityPlace, Name: pt.Name, Attributes: []timeline.Attribute{ { Name: "coordinate", Latitude: item.Location.Latitude, Longitude: item.Location.Longitude, Identity: true, // TODO: I feel like this should be just `Identifying: true`, but that creates "_entity" attributes... Metadata: timeline.Metadata{ "URL": pt.Link.Href, }, }, }, }) } } switch pt := l.Original.(type) { case trkpt: if _, ok := item.Metadata["Velocity"]; !ok { item.Metadata["Velocity"] = pt.Speed // use same key as with Google Location History } if _, ok := item.Metadata["Satellites"]; !ok { item.Metadata["Satellites"] = pt.Sat } item.Metadata = l.Metadata case wpt: makePlaceEntity(pt) } // in case a waypoint was clustered with other points, // iterate the points comprising it to see if there's any // place entities we can extract, so we don't skip them for _, cp := range l.ClusterPoints() { if pt, ok := cp.Original.(wpt); ok { makePlaceEntity(pt) } } if p.opt.Timeframe.ContainsItem(item, false) { return g, nil } } return nil, nil } // decoder wraps the XML decoder to get the next location from the document. // It tracks nesting state so we can be sure we're in the right part of the tree. type decoder struct { *xml.Decoder stack nesting metadataTime time.Time trkType string lastTrkPt trkpt } // NextLocation returns the next available point from the XML document. func (d *decoder) NextLocation(ctx context.Context) (*googlelocation.Location, error) { for { if err := ctx.Err(); err != nil { return nil, err } tkn, err := d.Token() if errors.Is(err, io.EOF) { break } if err != nil { return nil, fmt.Errorf("decoding next XML token: %w", err) } // TODO: grab this from the top in ? (we grab