/*
Timelinize
Copyright (c) 2024 Sergio Rubio
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
*/
// Package github implements a data source that imports GitHub starred repositories
// exported from the GitHub API.
//
// It expects a JSON file with an array of objects, each object representing a bookmark.
//
// The file should be named as follows:
// - ghstars.json
// - ghstars-.json
// - ghstars-.json
//
// The file format:
//
// [
// {
// "id": 841044067,
// "name": "timelinize",
// "html_url": "https://github.com/timelinize/timelinize",
// "description": "Store your data from all your accounts and devices in a single cohesive timeline on your own computer",
// "created_at": "2024-08-11T13:27:39Z",
// "updated_at": "2024-09-03T07:17:29Z",
// "pushed_at": "2024-09-02T15:31:59Z",
// "stargazers_count": 504,
// "language": "Go",
// "full_name": "timelinize/timelinize",
// "is_template": false,
// "topics": [
// "archival",
// "data-archiving",
// "data-import",
// "timeline"
// ],
// "private": false,
// "starred_at": "2024-08-12T17:55:48Z"
// }
// ]
//
// A tool to export starred repos to JSON and the exported format documentation
// can be found at https://github.com/rubiojr/gh-stars-exporter.
package github
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"regexp"
"time"
"github.com/timelinize/timelinize/timeline"
"go.uber.org/zap"
)
const (
DataSourceName = "GitHub"
DataSourceID = "github"
)
type Repository struct {
ID int `json:"id"`
Name string `json:"name"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
PushedAt time.Time `json:"pushed_at"`
StargazersCount int `json:"stargazers_count"`
Language string `json:"language"`
FullName string `json:"full_name"`
Topics []string `json:"topics"`
IsTemplate bool `json:"is_template"`
Private bool `json:"private"`
StarredAt time.Time `json:"starred_at"`
}
func init() {
err := timeline.RegisterDataSource(timeline.DataSource{
Name: DataSourceID,
Title: DataSourceName,
Icon: "github.svg",
NewFileImporter: func() timeline.FileImporter { return new(GitHub) },
})
if err != nil {
timeline.Log.Fatal("registering data source", zap.Error(err))
}
}
// GitHub interacts with the file system to get items.
type GitHub struct{}
// Recognize returns whether the input file is recognized.
func (GitHub) Recognize(_ context.Context, dirEntry timeline.DirEntry, _ timeline.RecognizeParams) (timeline.Recognition, error) {
isoDateRegexp := regexp.MustCompile(`^ghstars(-(\d{4}-\d{2}-\d{2}|[0-9]{10}))?.json$`)
// ghstars.json or ghstars-YYYY-MM-DD.json or ghstars-UNIX_TIMESTAMP.json
if isoDateRegexp.MatchString(dirEntry.Name()) {
return timeline.Recognition{Confidence: 1}, nil
}
return timeline.Recognition{}, nil
}
// FileImport conducts an import of the data using this data source.
func (c *GitHub) FileImport(ctx context.Context, dirEntry timeline.DirEntry, params timeline.ImportParams) error {
j, err := fs.ReadFile(dirEntry.FS, dirEntry.Filename)
if err != nil {
return err
}
var repos []*Repository
err = json.Unmarshal(j, &repos)
if err != nil {
return fmt.Errorf("malformed JSON: %w", err)
}
for _, repo := range repos {
// StarredAt is a required field.
if repo.StarredAt.IsZero() {
return fmt.Errorf("missing starred_at field for repo %s", repo.FullName)
}
// HTMLURL is a required field.
if repo.HTMLURL == "" {
return fmt.Errorf("missing HTMLURL field for repo %s", repo.FullName)
}
select {
// Callers can cancel the import.
case <-ctx.Done():
return ctx.Err()
default:
item := &timeline.Item{
Classification: timeline.ClassBookmark,
Timestamp: repo.StarredAt,
IntermediateLocation: dirEntry.Filename,
Metadata: timeline.Metadata{
"ID": repo.ID,
"Name": repo.Name,
"Full name": repo.FullName,
"URL": repo.HTMLURL,
"Description": repo.Description,
"Created": repo.CreatedAt,
"Updated": repo.UpdatedAt,
"Pushed": repo.PushedAt,
"Stargazers": repo.StargazersCount,
"Topics": repo.Topics,
"Language": repo.Language,
"Private": repo.Private,
},
}
params.Pipeline <- &timeline.Graph{Item: item}
}
}
return nil
}