pedro/rating-orama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update project and selectors

Browse Source
This commit is contained in:
Pedro Pérez
2025-01-28 17:07:05 +01:00
parent 430892a512
commit 34d1088d9d
15 changed files with 325 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/internal/app/app.go
+16
View File
@@ -0,0 +1,16 @@
package app
import (
"gopher-toolbox/app"
)
type ExtendedApp struct {
app.App
}
func NewExtendedApp(appName, version, envDirectory string) *ExtendedApp {
app := app.New(appName, version, envDirectory)
return &ExtendedApp{
App: *app,
}
}
core/internal/handlers/handlers.go
+4 -16
View File
@@ -1,30 +1,18 @@
package handlers
import (
"net/http"
"gopher-toolbox/app"
"github.com/gofiber/fiber/v2"
"github.com/zepyrshut/rating-orama/internal/app"
"github.com/zepyrshut/rating-orama/internal/repository"
)
type Handlers struct {
app *app.App
app *app.ExtendedApp
queries repository.ExtendedQuerier
}
func New(app *app.App, q repository.ExtendedQuerier) *Handlers {
func New(r repository.ExtendedQuerier, app *app.ExtendedApp) *Handlers {
return &Handlers{
app: app,
queries: q,
queries: r,
}
}
func (hq *Handlers) ToBeImplemented(c *fiber.Ctx) error {
return c.Status(http.StatusNotImplemented).JSON("not implemented")
}
func (hq *Handlers) Ping(c *fiber.Ctx) error {
return c.JSON("pong")
}
core/internal/handlers/tvshow.go
+5 -1
View File
@@ -13,6 +13,10 @@ import (
func (hq *Handlers) GetTVShow(c *fiber.Ctx) error {
ttShowID := c.Query("ttid")
if ttShowID == "" {
return c.SendStatus(http.StatusBadRequest)
}
var title string
var scraperEpisodes []scraper.Episode
var sqlcEpisodes []sqlc.Episode
@@ -20,7 +24,7 @@ func (hq *Handlers) GetTVShow(c *fiber.Ctx) error {
tvShow, err := hq.queries.CheckTVShowExists(c.Context(), ttShowID)
if err != nil {
title, scraperEpisodes = scraper.ScrapeEpisodes(ttShowID)
// TODO: make transactional
//TODO: make transactional
ttShow, err := hq.queries.CreateTVShow(c.Context(), sqlc.CreateTVShowParams{
TtImdb: ttShowID,
Name: title,
core/internal/repository/pgxrepo.go
+22 -17
View File
@@ -2,39 +2,44 @@ package repository
import (
"context"
"log/slog"
"fmt"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/zepyrshut/rating-orama/internal/app"
"github.com/zepyrshut/rating-orama/internal/sqlc"
)
type pgxRepository struct {
*sqlc.Queries
db *pgxpool.Pool
pool *pgxpool.Pool
app *app.ExtendedApp
}
func NewPGXRepo(db *pgxpool.Pool) ExtendedQuerier {
var _ ExtendedQuerier = &pgxRepository{}
func NewPGXRepo(pgx *pgxpool.Pool, app *app.ExtendedApp) ExtendedQuerier {
return &pgxRepository{
Queries: sqlc.New(db),
db: db,
Queries: sqlc.New(pgx),
pool: pgx,
app: app,
}
}
func (r *pgxRepository) execTx(ctx context.Context, txFunc func(tx pgx.Tx) error) error {
slog.Info("starting transaction", "txFunc", txFunc)
tx, err := r.db.Begin(ctx)
func (r *pgxRepository) execTx(ctx context.Context, fn func(*sqlc.Queries) error) error {
tx, err := r.pool.Begin(ctx)
if err != nil {
slog.Error("failed to start transaction", "error", err)
return err
}
defer tx.Rollback(ctx)
if err := txFunc(tx); err != nil {
slog.Error("failed to execute transaction", "error", err)
return err
}
slog.Info("committing transaction", "txFunc", txFunc)
q := sqlc.New(tx)
err = fn(q)
if err != nil {
if rbErr := tx.Rollback(ctx); rbErr != nil {
return fmt.Errorf("tx err: %v, rb err: %v", err, rbErr)
}
return err
}
return tx.Commit(ctx)
}
core/internal/repository/querier.go
+2 -2
View File
@@ -2,12 +2,12 @@ package repository
import (
"context"
"github.com/zepyrshut/rating-orama/internal/scraper"
"github.com/zepyrshut/rating-orama/internal/scraper"
"github.com/zepyrshut/rating-orama/internal/sqlc"
)
type ExtendedQuerier interface {
sqlc.Querier
CreateTvShowWithEpisodes(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error)
CreateTvShowWithEpisodesTX(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error)
}
core/internal/repository/tvshow.go
-33
View File
@@ -1,33 +0,0 @@
package repository
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/zepyrshut/rating-orama/internal/scraper"
"github.com/zepyrshut/rating-orama/internal/sqlc"
)
func (r *pgxRepository) CreateTvShowWithEpisodes(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error) {
var sqlcEpisodes []sqlc.Episode
err := r.execTx(ctx, func(tx pgx.Tx) error {
qtx := r.WithTx(tx)
tvShow, err := qtx.CreateTVShow(ctx, tvShow)
if err != nil {
return err
}
for _, episode := range episodes {
sqlcEpisodeParams := episode.ToEpisodeParams(tvShow.ID)
episode, err := qtx.CreateEpisodes(ctx, sqlcEpisodeParams)
if err != nil {
return err
}
sqlcEpisodes = append(sqlcEpisodes, episode)
}
return nil
})
return sqlcEpisodes, err
}
core/internal/repository/tvshow.tx.go
+34
View File
@@ -0,0 +1,34 @@
package repository
import (
"context"
"github.com/zepyrshut/rating-orama/internal/scraper"
"github.com/zepyrshut/rating-orama/internal/sqlc"
)
func (r *pgxRepository) CreateTvShowWithEpisodesTX(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error) {
var err error
var episodesSqlc []sqlc.Episode
err = r.execTx(ctx, func(tx *sqlc.Queries) error {
tvShow, err := tx.CreateTVShow(ctx, tvShow)
if err != nil {
return err
}
for _, episode := range episodes {
sqlcEpisodeParams := episode.ToEpisodeParams(tvShow.ID)
episode, err := tx.CreateEpisodes(ctx, sqlcEpisodeParams)
if err != nil {
return err
}
episodesSqlc = append(episodesSqlc, episode)
}
return nil
})
return episodesSqlc, err
}
core/internal/scraper/tvshow.go
+11 -24
View File
@@ -3,6 +3,7 @@ package scraper
import (
"fmt"
"log/slog"
"os"
"regexp"
"sort"
"strconv"
@@ -43,20 +44,6 @@ func (e Episode) ToEpisodeParams(tvShowID int32) sqlc.CreateEpisodesParams {
}
}
const (
titleSelector = "h2.sc-b8cc654b-9.dmvgRY"
seasonsSelector = "ul.ipc-tabs a[data-testid='tab-season-entry']"
episodeCardSelector = "article.sc-f8507e90-1.cHtpvn.episode-item-wrapper"
seasonEpisodeAndTitleSelector = "div.ipc-title__text"
releasedDateSelector = "span.sc-f2169d65-10.bYaARM"
plotSelector = "div.ipc-html-content-inner-div"
starRatingSelector = "span.ipc-rating-star--rating"
voteCountSelector = "span.ipc-rating-star--voteCount"
imdbEpisodesURL = "https://www.imdb.com/title/%s/episodes/?season=%d"
visitURL = "https://www.imdb.com/title/%s/episodes"
)
func ScrapeEpisodes(ttImdb string) (string, []Episode) {
c := colly.NewCollector(
colly.AllowedDomains("imdb.com", "www.imdb.com"),
@@ -70,7 +57,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) {
var seasons []int
var title string
c.OnHTML(seasonsSelector, func(e *colly.HTMLElement) {
c.OnHTML(os.Getenv("SEASON_SELECTOR"), func(e *colly.HTMLElement) {
seasonText := strings.TrimSpace(e.Text)
seasonNum, err := strconv.Atoi(seasonText)
if err == nil {
@@ -78,7 +65,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) {
}
})
c.OnHTML(titleSelector, func(e *colly.HTMLElement) {
c.OnHTML(os.Getenv("TITLE_SELECTOR"), func(e *colly.HTMLElement) {
title = e.Text
})
@@ -103,7 +90,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) {
})
for _, seasonNum := range uniqueSeasons {
seasonURL := fmt.Sprintf(imdbEpisodesURL, ttImdb, seasonNum)
seasonURL := fmt.Sprintf(os.Getenv("IMDB_EPISODES_URL"), ttImdb, seasonNum)
slog.Info("visiting season", "url", seasonURL)
_ = episodeCollector.Visit(seasonURL)
}
@@ -111,7 +98,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) {
episodeCollector.Wait()
})
_ = c.Visit(fmt.Sprintf(visitURL, ttImdb))
_ = c.Visit(fmt.Sprintf(os.Getenv("VISIT_URL"), ttImdb))
c.Wait()
slog.Info("scraped all seasons", "length", len(allSeasons))
@@ -126,26 +113,26 @@ func extractEpisodesFromSeason(data string) []Episode {
}
var episodes []Episode
doc.Find(episodeCardSelector).Each(func(i int, s *goquery.Selection) {
doc.Find(os.Getenv("EPISODE_CARD_SELECTOR")).Each(func(i int, s *goquery.Selection) {
var episode Episode
seasonEpisodeTitle := s.Find(seasonEpisodeAndTitleSelector).Text()
seasonEpisodeTitle := s.Find(os.Getenv("SEASON_EPISODE_AND_TITLE_SELECTOR")).Text()
episode.Season, episode.Episode, episode.Name = parseSeasonEpisodeTitle(seasonEpisodeTitle)
releasedDate := s.Find(releasedDateSelector).Text()
releasedDate := s.Find(os.Getenv("RELEASED_DATE_SELECTOR")).Text()
episode.Released = parseReleasedDate(releasedDate)
plot := s.Find(plotSelector).Text()
plot := s.Find(os.Getenv("PLOT_SELECTOR")).Text()
if plot == "Add a plot" {
episode.Plot = ""
} else {
episode.Plot = plot
}
starRating := s.Find(starRatingSelector).Text()
starRating := s.Find(os.Getenv("STAR_RATING_SELECTOR")).Text()
episode.Rate = parseStarRating(starRating)
voteCount := s.Find(voteCountSelector).Text()
voteCount := s.Find(os.Getenv("VOTE_COUNT_SELECTOR")).Text()
episode.VoteCount = parseVoteCount(voteCount)
episodes = append(episodes, episode)
core/internal/transfers/episodes.go
+9
View File
@@ -0,0 +1,9 @@
package transfers
type EpisodePayload struct {
Title string
Season int
Episode int
Description string
Rating float64
}