instances-api/main.go

362 lines
7.9 KiB
Go
Raw Normal View History

2022-02-14 07:35:02 +00:00
package main
import (
2022-02-22 14:13:23 +00:00
"context"
2023-02-20 11:06:21 +00:00
"encoding/json"
2023-07-21 16:03:29 +00:00
"errors"
"fmt"
2022-02-14 07:35:02 +00:00
"io"
"log"
"net/http"
2022-02-22 14:13:23 +00:00
"os"
2022-02-14 07:35:02 +00:00
"regexp"
"strconv"
"strings"
2023-07-21 16:03:29 +00:00
"sync"
2022-02-14 07:35:02 +00:00
"time"
2023-07-21 16:03:29 +00:00
"github.com/gofiber/fiber/v2"
2022-02-23 11:23:14 +00:00
"github.com/gofiber/fiber/v2/middleware/cors"
2022-02-14 07:35:02 +00:00
"github.com/gofiber/fiber/v2/middleware/etag"
2023-08-20 19:04:25 +00:00
"github.com/google/go-github/v54/github"
2022-02-22 14:13:23 +00:00
"golang.org/x/oauth2"
2022-02-14 07:35:02 +00:00
)
var monitored_instances = []Instance{}
var number_re = regexp.MustCompile(`(?m)(\d+)`)
type Instance struct {
Name string `json:"name"`
ApiUrl string `json:"api_url"`
Locations string `json:"locations"`
Version string `json:"version"`
UpToDate bool `json:"up_to_date"`
Cdn bool `json:"cdn"`
Registered int `json:"registered"`
LastChecked int64 `json:"last_checked"`
Cache bool `json:"cache"`
S3Enabled bool `json:"s3_enabled"`
ImageProxyUrl string `json:"image_proxy_url"`
RegistrationDisabled bool `json:"registration_disabled"`
2023-02-20 11:06:21 +00:00
}
type FrontendConfig struct {
S3Enabled bool `json:"s3Enabled"`
2023-08-15 18:13:26 +00:00
ImageProxyUrl string `json:"imageProxyUrl"`
RegistrationDisabled bool `json:"registrationDisabled"`
2022-02-14 07:35:02 +00:00
}
type PipedStreams struct {
Url string `json:"url"`
}
type Streams struct {
VideoStreams []PipedStreams `json:"videoStreams"`
}
2023-07-26 20:31:31 +00:00
var client = http.Client{
Timeout: 10 * time.Second,
}
func testUrl(url string) (*http.Response, error) {
2023-08-30 18:58:48 +00:00
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Piped-Instances-API/(https://github.com/TeamPiped/instances-api)")
resp, err := client.Do(req)
if err != nil {
return resp, err
}
if resp.StatusCode != 200 {
return resp, errors.New(fmt.Sprintf("Invalid response code at %s: %d", url, resp.StatusCode))
}
return resp, err
}
func testCaching(ApiUrl string) (bool, error) {
resp, err := testUrl(ApiUrl + "/trending?region=US")
if err != nil {
return false, err
}
oldTiming := resp.Header.Get("Server-Timing")
resp, err = testUrl(ApiUrl + "/trending?region=US")
if err != nil {
return false, err
}
newTiming := resp.Header.Get("Server-Timing")
cacheWorking := oldTiming == newTiming
return cacheWorking, nil
}
func getConfig(ApiUrl string) (FrontendConfig, error) {
resp, err := testUrl(ApiUrl + "/config")
if err != nil {
return FrontendConfig{}, err
}
bytes, err := io.ReadAll(resp.Body)
if err != nil {
return FrontendConfig{}, err
}
var config FrontendConfig
err = json.Unmarshal(bytes, &config)
if err != nil {
return FrontendConfig{}, err
}
return config, nil
}
func getStreams(ApiUrl string, VideoId string) (Streams, error) {
resp, err := testUrl(ApiUrl + "/streams/" + VideoId)
if err != nil {
return Streams{}, err
}
bytes, err := io.ReadAll(resp.Body)
if err != nil {
return Streams{}, err
}
var streams Streams
err = json.Unmarshal(bytes, &streams)
if err != nil {
return Streams{}, err
}
return streams, nil
}
2023-07-26 11:17:18 +00:00
func getInstanceDetails(split []string, latest string) (Instance, error) {
ApiUrl := strings.TrimSpace(split[1])
wg := sync.WaitGroup{}
errorChannel := make(chan error)
// the amount of tests to do
wg.Add(6)
var lastChecked int64
var registered int64
var config FrontendConfig
var hash string
var version string
var cacheWorking bool
go func() {
wg.Wait()
close(errorChannel)
}()
go func() {
defer wg.Done()
if _, err := testUrl(ApiUrl + "/healthcheck"); err != nil {
errorChannel <- err
return
2023-07-21 16:03:29 +00:00
}
lastChecked = time.Now().Unix()
}()
go func() {
defer wg.Done()
resp, err := testUrl(ApiUrl + "/registered/badge")
2023-07-21 16:03:29 +00:00
if err != nil {
errorChannel <- err
return
2023-07-21 16:03:29 +00:00
}
registered, err = strconv.ParseInt(number_re.FindString(resp.Request.URL.Path), 10, 32)
2023-07-21 16:03:29 +00:00
if err != nil {
errorChannel <- err
2023-07-21 16:03:29 +00:00
}
}()
go func() {
defer wg.Done()
resp, err := testUrl(ApiUrl + "/version")
2023-07-21 16:03:29 +00:00
if err != nil {
errorChannel <- err
return
2023-07-21 16:03:29 +00:00
}
buf := new(strings.Builder)
_, err = io.Copy(buf, resp.Body)
if err != nil {
errorChannel <- err
return
2023-07-21 16:03:29 +00:00
}
version = strings.TrimSpace(buf.String())
2023-07-21 16:03:29 +00:00
version_split := strings.Split(version, "-")
hash = version_split[len(version_split)-1]
}()
2023-07-21 16:03:29 +00:00
go func() {
defer wg.Done()
var err error
config, err = getConfig(ApiUrl)
2023-07-21 16:03:29 +00:00
if err != nil {
errorChannel <- err
2023-07-21 16:03:29 +00:00
}
}()
2023-07-21 16:03:29 +00:00
go func() {
defer wg.Done()
var err error
cacheWorking, err = testCaching(ApiUrl)
2023-07-21 16:03:29 +00:00
if err != nil {
errorChannel <- err
2023-07-21 16:03:29 +00:00
}
}()
2023-07-21 16:03:29 +00:00
go func() {
defer wg.Done()
2023-07-21 16:03:29 +00:00
// check if instance can fetch videos
streams, err := getStreams(ApiUrl, "jNQXAC9IVRw")
if err != nil {
errorChannel <- err
return
}
if len(streams.VideoStreams) == 0 {
errorChannel <- errors.New("no streams")
}
// head request to check first stream
if _, err := testUrl(streams.VideoStreams[0].Url); err != nil {
errorChannel <- err
2023-07-21 16:03:29 +00:00
}
}()
2023-07-21 16:03:29 +00:00
for err := range errorChannel {
return Instance{}, err
2023-07-21 16:03:29 +00:00
}
return Instance{
Name: strings.TrimSpace(split[0]),
ApiUrl: ApiUrl,
Locations: strings.TrimSpace(split[2]),
Cdn: strings.TrimSpace(split[3]) == "Yes",
Registered: int(registered),
LastChecked: lastChecked,
Version: version,
UpToDate: strings.Contains(latest, hash),
Cache: cacheWorking,
S3Enabled: config.S3Enabled,
ImageProxyUrl: config.ImageProxyUrl,
RegistrationDisabled: config.RegistrationDisabled,
}, nil
2023-07-21 16:03:29 +00:00
}
2022-02-14 07:35:02 +00:00
func monitorInstances() {
2022-02-22 14:13:23 +00:00
ctx := context.Background()
var tc *http.Client
if os.Getenv("GITHUB_TOKEN") != "" {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")},
)
tc = oauth2.NewClient(ctx, ts)
}
gh_client := github.NewClient(tc)
2022-02-14 07:35:02 +00:00
// do forever
for {
// send a request to get markdown from GitHub
2023-08-30 18:58:48 +00:00
req, err := http.NewRequest("GET", "https://raw.githubusercontent.com/wiki/TeamPiped/Piped-Frontend/Instances.md", nil)
if err != nil {
log.Print(err)
continue
}
req.Header.Set("User-Agent", "Piped-Instances-API/(https://github.com/TeamPiped/instances-api)")
resp, err := client.Do(req)
2022-02-14 07:35:02 +00:00
if err != nil {
log.Print(err)
continue
}
2022-02-22 14:13:23 +00:00
// Find Latest Commit from GitHub
var latest string
{
commits, _, err := gh_client.Repositories.ListCommits(ctx, "TeamPiped", "Piped-Backend", &github.CommitsListOptions{
ListOptions: github.ListOptions{
PerPage: 1,
},
})
if err != nil {
log.Print(err)
time.Sleep(time.Second * 5)
continue
}
latest = commits[0].GetSHA()
}
2022-02-14 07:35:02 +00:00
if resp.StatusCode == 200 {
// parse the response
buf := new(strings.Builder)
_, err := io.Copy(buf, resp.Body)
if err != nil {
log.Print(err)
continue
}
lines := strings.Split(buf.String(), "\n")
instancesMap := make(map[int]Instance)
2022-02-14 07:35:02 +00:00
2023-07-21 16:03:29 +00:00
wg := sync.WaitGroup{}
2023-07-26 11:17:18 +00:00
skipped := 0
checking := 0
2023-07-26 11:17:18 +00:00
for _, line := range lines {
split := strings.Split(line, "|")
if len(split) < 5 {
continue
}
// skip first two table lines
if skipped < 2 {
skipped++
2023-07-21 16:03:29 +00:00
continue
}
2023-02-20 11:06:21 +00:00
2023-07-21 16:03:29 +00:00
wg.Add(1)
go func(i int, split []string) {
defer wg.Done()
2023-07-26 11:17:18 +00:00
instance, err := getInstanceDetails(split, latest)
2023-07-21 16:03:29 +00:00
if err == nil {
instancesMap[i] = instance
2023-07-21 16:03:29 +00:00
} else {
log.Print(err)
}
}(checking, split)
checking++
2022-02-14 07:35:02 +00:00
}
2023-07-21 16:03:29 +00:00
wg.Wait()
2022-02-14 07:35:02 +00:00
// Map to ordered array
var instances []Instance
for i := 0; i < checking; i++ {
instance, ok := instancesMap[i]
if ok {
instances = append(instances, instance)
}
}
2022-02-14 07:35:02 +00:00
// update the global instances variable
monitored_instances = instances
}
_ = resp.Body.Close()
2022-02-14 07:35:02 +00:00
time.Sleep(time.Minute)
}
}
func main() {
go monitorInstances()
2022-02-22 14:13:23 +00:00
app := fiber.New()
2022-02-23 11:23:14 +00:00
app.Use(cors.New())
2022-02-14 07:35:02 +00:00
app.Use(etag.New())
app.Get("/", func(c *fiber.Ctx) error {
return c.JSON(monitored_instances)
})
err := app.Listen(":3000")
if err != nil {
panic(err)
}
2022-02-14 07:35:02 +00:00
}