/*
Timelinize
Copyright (c) 2013 Matthew Holt
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
*/
// Package gpx implements a data source for GPS Exchange Format (https://en.wikipedia.org/wiki/GPS_Exchange_Format).
package gpx
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"io/fs"
"path"
"strings"
"time"
"github.com/timelinize/timelinize/datasources/googlelocation"
"github.com/timelinize/timelinize/timeline"
"go.uber.org/zap"
"golang.org/x/net/html/charset"
)
func init() {
err := timeline.RegisterDataSource(timeline.DataSource{
Name: "gpx",
Title: "GPS Exchange",
Icon: "gpx.svg",
Description: "A .gpx file or folder of .gpx files containing location tracks",
NewOptions: func() any { return new(Options) },
NewFileImporter: func() timeline.FileImporter { return new(FileImporter) },
})
if err != nil {
timeline.Log.Fatal("registering data source", zap.Error(err))
}
}
// Options configures the data source.
type Options struct {
// The ID of the owner entity. REQUIRED for linking entity in DB.
// TODO: maybe an attribute ID instead, in case the data represents multiple people
OwnerEntityID uint64 `json:"owner_entity_id"`
// Options specific to the location processor.
googlelocation.LocationProcessingOptions
}
// FileImporter implements the timeline.FileImporter interface.
type FileImporter struct{}
// Recognize returns whether the file is supported.
func (FileImporter) Recognize(_ context.Context, dirEntry timeline.DirEntry, _ timeline.RecognizeParams) (timeline.Recognition, error) {
rec := timeline.Recognition{DirThreshold: .9}
// we can import directories, but let the import planner figure that out; only recognize files
if dirEntry.IsDir() {
return rec, nil
}
// recognize by file extension
if strings.ToLower(path.Ext(dirEntry.Name())) == ".gpx" {
rec.Confidence = 1
}
return rec, nil
}
// FileImport imports data from a file or folder.
func (fi *FileImporter) FileImport(ctx context.Context, dirEntry timeline.DirEntry, params timeline.ImportParams) error {
dsOpt := params.DataSourceOptions.(*Options)
owner := timeline.Entity{
ID: dsOpt.OwnerEntityID,
}
return fs.WalkDir(dirEntry.FS, dirEntry.Filename, func(fpath string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
if strings.HasPrefix(d.Name(), ".") {
// skip hidden files & folders
if d.IsDir() {
return fs.SkipDir
}
return nil
}
if d.IsDir() {
return nil // traverse into subdirectories
}
// skip unsupported file types
if ext := strings.ToLower(path.Ext(d.Name())); ext != ".gpx" {
return nil
}
file, err := dirEntry.FS.Open(fpath)
if err != nil {
return err
}
defer file.Close()
proc, err := NewProcessor(file, owner, params, dsOpt.LocationProcessingOptions)
if err != nil {
return err
}
for {
g, err := proc.NextGPXGraph(ctx)
if err != nil {
return err
}
if g == nil {
break
}
params.Pipeline <- g
}
return nil
})
}
// Processor can get the next GPX point. Call NewProcessor to make a valid instance.
type Processor struct {
dec *decoder
opt timeline.ImportParams
locProc googlelocation.LocationSource
owner timeline.Entity
}
// NewProcessor returns a new GPX processor.
func NewProcessor(file io.Reader, owner timeline.Entity, opt timeline.ImportParams, locOpt googlelocation.LocationProcessingOptions) (*Processor, error) {
// create XML decoder (wrapped to track some state as it decodes)
xmlDec := &decoder{Decoder: xml.NewDecoder(file)}
xmlDec.CharsetReader = charset.NewReaderLabel // handle non-UTF-8 encodings
// create location processor to clean up any noisy raw data
locProc, err := googlelocation.NewLocationProcessor(xmlDec, locOpt)
if err != nil {
return nil, err
}
return &Processor{
dec: xmlDec,
opt: opt,
locProc: locProc,
owner: owner,
}, nil
}
// NextGPXGraph returns the next GPX graph (item or entity)
func (p *Processor) NextGPXGraph(ctx context.Context) (*timeline.Graph, error) {
for {
if err := ctx.Err(); err != nil {
return nil, err
}
l, err := p.locProc.NextLocation(ctx)
if err != nil {
return nil, err
}
if l == nil {
break
}
if l.Metadata == nil {
l.Metadata = make(timeline.Metadata)
}
item := &timeline.Item{
Classification: timeline.ClassLocation,
Timestamp: l.Timestamp,
Timespan: l.Timespan,
Location: l.Location(),
Owner: p.owner,
Metadata: l.Metadata,
}
g := &timeline.Graph{Item: item}
makePlaceEntity := func(pt wpt) {
// store named waypoints as place entities
if pt.Name != "" {
g.ToEntity(timeline.RelVisit, &timeline.Entity{
Type: timeline.EntityPlace,
Name: pt.Name,
Attributes: []timeline.Attribute{
{
Name: "coordinate",
Latitude: item.Location.Latitude,
Longitude: item.Location.Longitude,
Identity: true, // TODO: I feel like this should be just `Identifying: true`, but that creates "_entity" attributes...
Metadata: timeline.Metadata{
"URL": pt.Link.Href,
},
},
},
})
}
}
switch pt := l.Original.(type) {
case trkpt:
if _, ok := item.Metadata["Velocity"]; !ok {
item.Metadata["Velocity"] = pt.Speed // use same key as with Google Location History
}
if _, ok := item.Metadata["Satellites"]; !ok {
item.Metadata["Satellites"] = pt.Sat
}
item.Metadata = l.Metadata
case wpt:
makePlaceEntity(pt)
}
// in case a waypoint was clustered with other points,
// iterate the points comprising it to see if there's any
// place entities we can extract, so we don't skip them
for _, cp := range l.ClusterPoints() {
if pt, ok := cp.Original.(wpt); ok {
makePlaceEntity(pt)
}
}
if p.opt.Timeframe.ContainsItem(item, false) {
return g, nil
}
}
return nil, nil
}
// decoder wraps the XML decoder to get the next location from the document.
// It tracks nesting state so we can be sure we're in the right part of the tree.
type decoder struct {
*xml.Decoder
stack nesting
metadataTime time.Time
trkType string
lastTrkPt trkpt
}
// NextLocation returns the next available point from the XML document.
func (d *decoder) NextLocation(ctx context.Context) (*googlelocation.Location, error) {
for {
if err := ctx.Err(); err != nil {
return nil, err
}
tkn, err := d.Token()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("decoding next XML token: %w", err)
}
// TODO: grab this from the top in ? (we grab