From 0621fbfbf19f7529ba092e800a89cfb3a7aed229 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 8 Nov 2021 20:08:56 +0000 Subject: [PATCH] go/streetworks: init, schedule on totoro --- go/default.nix | 1 + go/streetworks/default.nix | 12 + go/streetworks/streetworks.go | 295 ++++++++++++++++++ ops/nixos/totoro/default.nix | 16 + .../github.com/google/go-cmp/default.nix | 14 + 5 files changed, 338 insertions(+) create mode 100644 go/streetworks/default.nix create mode 100644 go/streetworks/streetworks.go create mode 100644 third_party/gopkgs/github.com/google/go-cmp/default.nix diff --git a/go/default.nix b/go/default.nix index 33a3520db8..a3a089f017 100644 --- a/go/default.nix +++ b/go/default.nix @@ -6,4 +6,5 @@ args: { twitterchiver = import ./twitterchiver args; twitternuke = import ./twitternuke args; minotarproxy = import ./minotarproxy args; + streetworks = import ./streetworks args; } diff --git a/go/streetworks/default.nix b/go/streetworks/default.nix new file mode 100644 index 0000000000..b2fe7e957b --- /dev/null +++ b/go/streetworks/default.nix @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2020 Luke Granger-Brown +# +# SPDX-License-Identifier: Apache-2.0 + +{ depot, ... }: +depot.third_party.buildGo.program { + name = "streetworks"; + srcs = [ ./streetworks.go ]; + deps = with depot.third_party; [ + gopkgs."github.com".google.go-cmp.cmp + ]; +} diff --git a/go/streetworks/streetworks.go b/go/streetworks/streetworks.go new file mode 100644 index 0000000000..52f2f9f499 --- /dev/null +++ b/go/streetworks/streetworks.go @@ -0,0 +1,295 @@ +package main + +import ( + "context" + "encoding/json" + "encoding/xml" + "flag" + "fmt" + "io" + "log" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/google/go-cmp/cmp" +) + +var ( + postcode = flag.String("postcode", "", "Postcode to look for") + pushoverToken = flag.String("pushover_token", "", "Pushover application token") + pushoverUser = flag.String("pushover_user", "", "Pushover user token") + + checkIntervalError = flag.Duration("interval_error", 5*time.Minute, "Interval between loop reruns in the event of an error") + checkInterval = flag.Duration("interval", 30*time.Minute, "Interval between checks") +) + +type Position struct { + Lat float64 `xml:"lat,attr"` + Lon float64 `xml:"lng,attr"` +} + +func (p1 Position) Equal(p2 Position) bool { + return p1.Lat == p2.Lat && p1.Lon == p2.Lon +} + +type Locations struct { + XMLName xml.Name `xml:"Locations"` + + Postcode string `xml:"postcode,attr"` + Position + + StreetWorks []StreetWork `xml:"StreetWorks"` +} + +// sortIfUnsorted returns the input slice if it's sorted, otherwise it creates a *new* slice that _is_ sorted. +func sortIfUnsorted(sw []StreetWork) []StreetWork { + less := func(i, j int) bool { + a, b := sw[i], sw[j] + switch { + case a.StartDate.Before(b.StartDate): + return true + case a.StartDate.After(b.StartDate): + return false + case a.Street < b.Street: + return true + case a.Street > b.Street: + return false + case a.Location < b.Location: + return true + default: + return false + } + } + + if sort.SliceIsSorted(sw, less) { + return sw + } + + // Make a brand new slice... + sw = append([]StreetWork(nil), sw...) + sort.Slice(sw, less) + return sw +} + +func (l1 Locations) Equal(l2 Locations) bool { + if l1.Postcode != l2.Postcode { + return false + } + + // Ignore Position, we don't care. + + if len(l1.StreetWorks) != len(l2.StreetWorks) { + return false + } + + l1s := sortIfUnsorted(l1.StreetWorks) + l2s := sortIfUnsorted(l2.StreetWorks) + + for n := range l1s { + a, b := l1s[n], l2s[n] + if !a.Equal(b) { + return false + } + } + return true +} + +type CamdenTime struct { + time.Time +} + +var CamdenTZ = func() *time.Location { + l, err := time.LoadLocation("Europe/London") + if err != nil { + panic(fmt.Errorf("loading Europe/London timezone: %v", err)) + } + return l +}() + +func (t *CamdenTime) UnmarshalText(data []byte) error { + rt, err := time.ParseInLocation("2006-01-02T15:04:05", string(data), CamdenTZ) + if err == nil { + t.Time = rt + } + return err +} + +func (t CamdenTime) Equal(t2 CamdenTime) bool { return t.Time.Equal(t2.Time) } +func (t CamdenTime) Before(t2 CamdenTime) bool { return t.Time.Before(t2.Time) } +func (t CamdenTime) After(t2 CamdenTime) bool { return t.Time.After(t2.Time) } + +type StreetWork struct { + Location string `xml:"Location,attr"` + StartDate CamdenTime `xml:"StartDate,attr"` // RFC3339 + EndDate CamdenTime `xml:"EndDate,attr"` // RFC3339 + LocalAuthorityRef string `xml:"LAref,attr"` + Organisation string `xml:"Organisation,attr"` + ExternalRef string `xml:"externalref,attr"` + Telephone string `xml:"Telephone,attr"` + Position + Description string `xml:"Description,attr"` + Street string `xml:"Street,attr"` + Type string `xml:"Type,attr"` + OutOfHoursTelephone string `xml:"OutofHours,attr"` +} + +func (sw1 StreetWork) Equal(sw2 StreetWork) bool { + for _, b := range []bool{ + sw1.Location == sw2.Location, + sw1.StartDate.Equal(sw2.StartDate), + sw1.EndDate.Equal(sw2.EndDate), + sw1.LocalAuthorityRef == sw2.LocalAuthorityRef, + sw1.Organisation == sw2.Organisation, + sw1.ExternalRef == sw2.ExternalRef, + sw1.Telephone == sw2.Telephone, + sw1.Position.Equal(sw2.Position), + sw1.Description == sw2.Description, + sw1.Street == sw2.Street, + sw1.Type == sw2.Type, + sw1.OutOfHoursTelephone == sw2.OutOfHoursTelephone, + } { + if !b { + return false + } + } + return true +} + +func getStreetWorks(ctx context.Context, postcode string) (*Locations, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", "https://find.camden.gov.uk/StreetWorksrest.aspx", nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequest: %w", err) + } + v := url.Values{} + v.Set("area", postcode) + req.URL.RawQuery = v.Encode() + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("making HTTP request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad HTTP response code %v (%v)", resp.StatusCode, resp.Status) + } + + l := new(Locations) + if err := xml.NewDecoder(resp.Body).Decode(l); err != nil { + return nil, fmt.Errorf("decoding XML response: %w", err) + } + if bs, err := io.ReadAll(resp.Body); err != nil { + return nil, fmt.Errorf("checking for trailing data: %w", err) + } else if len(bs) > 0 { + return nil, fmt.Errorf("found trailing data (%d bytes; %q)", len(bs), string(bs)) + } + + return l, nil +} + +type Notification struct { + Message string + Title string + Priority int + Sound string +} + +func (n Notification) EncodeValues() url.Values { + v := url.Values{} + v.Set("message", n.Message) + if n.Title != "" { + v.Set("title", n.Title) + } + if n.Priority != 0 { + v.Set("priority", strconv.Itoa(n.Priority)) + } + if n.Sound != "" { + v.Set("sound", n.Sound) + } + return v +} + +func sendPushoverNotification(ctx context.Context, token, user string, notification Notification) error { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + v := notification.EncodeValues() + v.Set("token", token) + v.Set("user", user) + + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.pushover.net/1/messages.json", strings.NewReader(v.Encode())) + if err != nil { + return fmt.Errorf("creating new HTTP request: %w", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("doing HTTP request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad HTTP response: %v (%q)", resp.StatusCode, resp.Status) + } + return nil +} + +func main() { + flag.Parse() + + if *postcode == "" { + log.Fatalf("-postcode must be provided.") + } + + ctx := context.Background() + + var lastWorks *Locations + + for { + l, err := getStreetWorks(ctx, *postcode) + if err != nil { + log.Printf("getStreetWorks failed: %v; waiting 5 minutes", err) + time.Sleep(*checkIntervalError) + continue + } + log.Println("refreshed data...") + + if lastWorks != nil && !lastWorks.Equal(*l) { + // OMG!!! + lj, err := json.MarshalIndent(l, "", " ") + if err != nil { + log.Fatalf("json.Marshal streetworks: %v", err) + } + + log.Println("it's changed!") + log.Println(string(lj)) + + diff := cmp.Diff(lastWorks.StreetWorks, l.StreetWorks) + if diff != "" { + log.Println(diff) + if *pushoverToken != "" && *pushoverUser != "" { + if err := sendPushoverNotification(ctx, *pushoverToken, *pushoverUser, Notification{ + Title: fmt.Sprintf("Camden StreetWorks Diff for %v", *postcode), + Message: diff, + Priority: -1, + }); err != nil { + log.Printf("sendPushoverNotification: %v; waiting 5 minutes", err) + time.Sleep(*checkIntervalError) + continue + // We'll see the diff again next time. + } + } + } + } + lastWorks = l + + time.Sleep(*checkInterval) + } +} diff --git a/ops/nixos/totoro/default.nix b/ops/nixos/totoro/default.nix index b6ddc5e4ea..1365ff187e 100644 --- a/ops/nixos/totoro/default.nix +++ b/ops/nixos/totoro/default.nix @@ -471,6 +471,22 @@ in { }; }; + systemd.services.streetworks = { + enable = true; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${depot.go.streetworks}/bin/streetworks -postcode='NW5 4HS' -pushover_token='${secrets.pushover.tokens.depot}' -pushover_user='${secrets.pushover.userKey}'"; + DynamicUser = true; + MountAPIVFS = true; + PrivateTmp = true; + PrivateUsers = true; + ProtectControlGroups = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + }; + }; + my.prometheus.additionalExporterPorts.trains = 2112; system.stateVersion = "20.03"; diff --git a/third_party/gopkgs/github.com/google/go-cmp/default.nix b/third_party/gopkgs/github.com/google/go-cmp/default.nix new file mode 100644 index 0000000000..049b733cbd --- /dev/null +++ b/third_party/gopkgs/github.com/google/go-cmp/default.nix @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2020 Luke Granger-Brown +# +# SPDX-License-Identifier: Apache-2.0 + +{ depot, ... }: +depot.third_party.buildGo.external { + path = "github.com/google/go-cmp"; + src = depot.third_party.nixpkgs.fetchFromGitHub { + owner = "google"; + repo = "go-cmp"; + rev = "6faefd0594fae82639a62c23f0aed1451509dcc0"; + hash = "sha256:0w0nyaqqd29bdk919g0r3xr1hpn32sfwbf59ai6z6ngdhjp95nbv"; + }; +}