From a8a8480e3cd781994c64dc4775f501643251de01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=BClsmann?= Date: Sat, 10 Feb 2024 12:05:36 +0100 Subject: [PATCH 1/6] theateraachen adapter --- go.mod | 5 +++-- go.sum | 6 ++++++ theateraachen.go | 24 ++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 theateraachen.go diff --git a/go.mod b/go.mod index 9763513..b106b4f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cbix/ics go 1.21.6 require ( - github.com/arran4/golang-ical v0.2.3 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/arran4/golang-ical v0.2.3 + github.com/coreos/go-systemd/v22 v22.5.0 + golang.org/x/net v0.21.0 ) diff --git a/go.sum b/go.sum index 0dd9286..ec50f2b 100644 --- a/go.sum +++ b/go.sum @@ -4,16 +4,22 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/theateraachen.go b/theateraachen.go new file mode 100644 index 0000000..0d46636 --- /dev/null +++ b/theateraachen.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "io" + "net/http" + + "golang.org/x/net/html" +) + +const ( + theateraachenApi = "https://www.theateraachen.de/de/spielplan/kalender.html?ajax=1&offset=%d" +) + +func theateraachenAdapter(w io.Writer) error { + for i := 0; i < 10; i++ { + res, err := http.Get(fmt.Sprintf(theateraachenApi, i)) + if err != nil { + return err + } + defer res.Body.Close() + } + return nil +} From 5683424b89111a9cb2d8ce71fc573b3f805a9679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=BClsmann?= Date: Sat, 10 Feb 2024 13:27:25 +0100 Subject: [PATCH 2/6] parse html --- .gitignore | 1 + theateraachen.go | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index da0a2c4..6ed89e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /ics *.html *.ics +/tmp/ diff --git a/theateraachen.go b/theateraachen.go index 0d46636..c5ec257 100644 --- a/theateraachen.go +++ b/theateraachen.go @@ -8,17 +8,41 @@ import ( "golang.org/x/net/html" ) +func init() { + RegisterAdapter("theateraachen", theateraachenUrl, "Theatre in Aachen", theateraachenAdapter) +} + const ( - theateraachenApi = "https://www.theateraachen.de/de/spielplan/kalender.html?ajax=1&offset=%d" + theateraachenUrl = "https://www.theateraachen.de/" + //theateraachenSrc = "https://www.theateraachen.de/de/spielplan/kalender.html?ajax=1&offset=%d" + theateraachenSrc = "https://theateraachen.reservix.de/events/%d" ) func theateraachenAdapter(w io.Writer) error { - for i := 0; i < 10; i++ { - res, err := http.Get(fmt.Sprintf(theateraachenApi, i)) + n := 0 + for i := 1; i < 10; i++ { + res, err := http.Get(fmt.Sprintf(theateraachenSrc, i)) if err != nil { return err } - defer res.Body.Close() + nodes, err := html.Parse(res.Body) + res.Body.Close() + if err != nil { + return err + } + startNode := nodes.FirstChild.FirstChild.NextSibling.FirstChild + if startNode.FirstChild == nil { + break + } + //fmt.Fprintf(w, "startNode: %+v\n", startNode) + for node := startNode; node != nil; node = node.NextSibling { + if node.FirstChild == nil || node.FirstChild.NextSibling == nil { + continue + } + n++ + fmt.Fprintf(w, "child: %+v\n", node.FirstChild.NextSibling) + } } + fmt.Fprintf(w, "nodes: %d\n", n) return nil } From 37dfa3fa984f18cb84a1bb50994a17a3825b6207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=BClsmann?= Date: Sat, 10 Feb 2024 15:13:40 +0100 Subject: [PATCH 3/6] parse reservix impressions --- theateraachen.go | 102 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 19 deletions(-) diff --git a/theateraachen.go b/theateraachen.go index c5ec257..a102145 100644 --- a/theateraachen.go +++ b/theateraachen.go @@ -1,48 +1,112 @@ package main import ( + "bufio" + "encoding/json" "fmt" "io" "net/http" + "strings" + "time" - "golang.org/x/net/html" + ics "github.com/arran4/golang-ical" ) func init() { - RegisterAdapter("theateraachen", theateraachenUrl, "Theatre in Aachen", theateraachenAdapter) + RegisterAdapter("theateraachen", theateraachenUrl, "Theatre in Aachen (beta)", theateraachenAdapter) +} + +type theateraachenData struct { + Ecommerce theateraachenEcommerce `json:"ecommerce"` +} + +type theateraachenEcommerce struct { + Impressions []theateraachenImpression `json:"impressions"` +} + +type theateraachenImpression struct { + Id string `json:"id"` + Name string `json:"name"` + Variant theateraachenVariant `json:"variant"` + Position int `json:"position"` +} + +type theateraachenVariant struct { + DateTime string `json:"dateTime"` + VenueName string `json:"venueName"` + GenreName string `json:"genreName"` +} + +func (i theateraachenImpression) StartTime() time.Time { + t, _ := time.ParseInLocation(theateraachenLocalFormat, i.Variant.DateTime, theateraachenTZ) + return t +} + +func (i theateraachenImpression) EndTime() time.Time { + return i.StartTime().Add(2 * time.Hour) } const ( - theateraachenUrl = "https://www.theateraachen.de/" - //theateraachenSrc = "https://www.theateraachen.de/de/spielplan/kalender.html?ajax=1&offset=%d" - theateraachenSrc = "https://theateraachen.reservix.de/events/%d" + theateraachenUrl = "https://www.theateraachen.de/" + theateraachenSrc = "https://theateraachen.reservix.de/events/%d" + theateraachenJsonPrefix = "var impressionData = " + theateraachenLocalFormat = "2006-01-02 15:04:05" ) +var theateraachenTZ, _ = time.LoadLocation("Europe/Berlin") + func theateraachenAdapter(w io.Writer) error { - n := 0 + cal := ics.NewCalendarFor("github.com/cbix/ics") + cal.SetName("Theater Aachen") + cal.SetUrl(theateraachenUrl) + cal.SetTimezoneId("Europe/Berlin") + now := time.Now() + pos := 0 +page: for i := 1; i < 10; i++ { - res, err := http.Get(fmt.Sprintf(theateraachenSrc, i)) + req, err := http.NewRequest("GET", fmt.Sprintf(theateraachenSrc, i), nil) if err != nil { return err } - nodes, err := html.Parse(res.Body) - res.Body.Close() + req.Header.Set("User-Agent", "Mozilla/5.0 Gecko/20100101 Firefox/122.0") + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + scanner := bufio.NewScanner(res.Body) + var jsonData []byte + for scanner.Scan() { + l := strings.Trim(scanner.Text(), " ;") + if strings.HasPrefix(l, theateraachenJsonPrefix) { + jsonData = []byte(strings.TrimPrefix(l, theateraachenJsonPrefix)) + break + } + } + if len(jsonData) == 0 { + continue + } + var data theateraachenData + err = json.Unmarshal(jsonData, &data) if err != nil { return err } - startNode := nodes.FirstChild.FirstChild.NextSibling.FirstChild - if startNode.FirstChild == nil { + if len(data.Ecommerce.Impressions) == 0 { break } - //fmt.Fprintf(w, "startNode: %+v\n", startNode) - for node := startNode; node != nil; node = node.NextSibling { - if node.FirstChild == nil || node.FirstChild.NextSibling == nil { - continue + for _, ev := range data.Ecommerce.Impressions { + p := ev.Position + if p <= pos { + break page } - n++ - fmt.Fprintf(w, "child: %+v\n", node.FirstChild.NextSibling) + pos = p + event := cal.AddEvent(fmt.Sprintf("%s@theateraachen.reservix.de", ev.Id)) + event.SetDtStampTime(now) + event.SetSummary(ev.Name) + event.SetDescription(ev.Variant.GenreName) + event.SetLocation(ev.Variant.VenueName) + event.SetStartAt(ev.StartTime()) + event.SetEndAt(ev.EndTime()) } } - fmt.Fprintf(w, "nodes: %d\n", n) - return nil + return cal.SerializeTo(w) } From ed82f5d8edb811218bc672748139221fd6f445bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=BClsmann?= Date: Sat, 10 Feb 2024 15:15:51 +0100 Subject: [PATCH 4/6] go mod tidy --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index b106b4f..3d89765 100644 --- a/go.mod +++ b/go.mod @@ -5,5 +5,4 @@ go 1.21.6 require ( github.com/arran4/golang-ical v0.2.3 github.com/coreos/go-systemd/v22 v22.5.0 - golang.org/x/net v0.21.0 ) diff --git a/go.sum b/go.sum index ec50f2b..de3e61e 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 10df5957388e4422eb126b6e9528883f631d3e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=BClsmann?= Date: Sat, 10 Feb 2024 15:19:29 +0100 Subject: [PATCH 5/6] cleanup --- theateraachen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theateraachen.go b/theateraachen.go index a102145..f14ba82 100644 --- a/theateraachen.go +++ b/theateraachen.go @@ -9,7 +9,7 @@ import ( "strings" "time" - ics "github.com/arran4/golang-ical" + "github.com/arran4/golang-ical" ) func init() { From 9f019e65d7f89e77d4a9fe3458a006ca4660719b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=BClsmann?= Date: Sat, 10 Feb 2024 16:43:24 +0100 Subject: [PATCH 6/6] report connection errors --- theateraachen.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/theateraachen.go b/theateraachen.go index f14ba82..cd8370a 100644 --- a/theateraachen.go +++ b/theateraachen.go @@ -3,6 +3,7 @@ package main import ( "bufio" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -58,6 +59,7 @@ var theateraachenTZ, _ = time.LoadLocation("Europe/Berlin") func theateraachenAdapter(w io.Writer) error { cal := ics.NewCalendarFor("github.com/cbix/ics") cal.SetName("Theater Aachen") + cal.SetColor("#f0ff3c") cal.SetUrl(theateraachenUrl) cal.SetTimezoneId("Europe/Berlin") now := time.Now() @@ -73,17 +75,22 @@ page: if err != nil { return err } + if res.StatusCode != http.StatusOK { + return errors.New(fmt.Sprintf("received status %s", res.Status)) + } scanner := bufio.NewScanner(res.Body) var jsonData []byte + found := false for scanner.Scan() { l := strings.Trim(scanner.Text(), " ;") if strings.HasPrefix(l, theateraachenJsonPrefix) { jsonData = []byte(strings.TrimPrefix(l, theateraachenJsonPrefix)) + found = true break } } - if len(jsonData) == 0 { - continue + if !found { + return errors.New("Expected JSON data not found") } var data theateraachenData err = json.Unmarshal(jsonData, &data)