You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
140 lines
4.1 KiB
Python
140 lines
4.1 KiB
Python
"""
|
|
Scrapes the Texas lottery website for scratch-off ticket
|
|
data and calculates the expected value for each game.
|
|
|
|
Texas publishes the number of tickets printed and how many
|
|
tickets are printed at each prize level.
|
|
|
|
We can calculated the expected value of a game by summing
|
|
the value of all the prizes and dividing that by the cost
|
|
of all the tickets.
|
|
|
|
The texas lottery website has an "top prizes remaining" or an "index" page that
|
|
has links to every game that could still be profitable.
|
|
Each individual game has a section for the "game rules" page and a prize table.
|
|
We can use each individual game page to gather the important data, and
|
|
then run our calculations.
|
|
|
|
Website that we'll be scraping:
|
|
http://www.txlottery.org/export/sites/lottery/Games/Scratch_Offs/all.html
|
|
|
|
Example usage:
|
|
python -m texas
|
|
Or:
|
|
LOGLEVEL=DEBUG USE_CACHE=True python -m texas
|
|
|
|
The following behavior is configurable through shell environment variables.
|
|
|
|
Set LOGLEVEL to print useful debug info to console.
|
|
LOGLEVEL=[DEBUG,INFO,WARNING,ERROR,CRITICAL]
|
|
Defaults to WARNING.
|
|
|
|
Set USE_CACHE to cache responses. This speeds up development
|
|
and is nice to the servers we're hitting.
|
|
USE_CACHE=[True]
|
|
Defaults to False. Note: Setting this env variable to the string False
|
|
will cause it to use cache because the string "False" evaluates to Truthy.
|
|
Either set it to True or don't set it.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
from xmlrpc import client
|
|
|
|
from bs4 import BeautifulSoup as bs
|
|
import pandas as pd
|
|
import requests
|
|
from lottery_data_scraper.schemas import GameSchema
|
|
from lottery_data_scraper.util import fetch_html
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BASE_URL = "http://www.txlottery.org"
|
|
INDEX_URL = (
|
|
"http://www.txlottery.org/export/sites/lottery/Games/Scratch_Offs/all.html"
|
|
)
|
|
|
|
|
|
def parse_index(html):
|
|
soup = bs(html, "lxml")
|
|
table = soup.find("table")
|
|
game_hrefs = table.select("tr > td > a")
|
|
game_urls = list(map(lambda x: BASE_URL + x.attrs["href"], game_hrefs))
|
|
return game_urls
|
|
|
|
|
|
def parse_game(url, html):
|
|
soup = bs(html, "lxml")
|
|
price = int(
|
|
re.match(
|
|
r"\$(\d+)",
|
|
soup.select("h3 > img")[0].attrs["alt"]
|
|
).group(1)
|
|
)
|
|
game_details = soup.select(".large-4.cell > h3")[0].parent.text.strip()
|
|
title = soup.select(".large-12.cell > .text-center > h2")[0].text.split(" - ")
|
|
name = title[1]
|
|
num = title[0][-4:]
|
|
num_tx = int(
|
|
re.match(
|
|
r".*?([\d,]+)",
|
|
soup.find(string=re.compile(r"There are approximately [\d,]+.*")).strip()
|
|
).group(1).replace(",", "")
|
|
)
|
|
# Prizes
|
|
table = soup.find("table")
|
|
df = pd.read_html(str(table))[0]
|
|
df = df.replace("---", 0)
|
|
df.iloc[:, 0] = df.iloc[:, 0].str.replace("$", "")
|
|
prizes = []
|
|
for prize, total, claimed in [list(r[1]) for r in df.iterrows()]:
|
|
match = re.match(r"\$?([\d,]+).*wk.*", prize)
|
|
if match:
|
|
value = float(match.group(1).replace(",", "")) * 20 * 52
|
|
prize = match.group(0)
|
|
else:
|
|
value = float(prize.replace(",", ""))
|
|
prize = "$" + prize
|
|
prizes.append(
|
|
{
|
|
"prize": prize,
|
|
"value": value,
|
|
"claimed": int(claimed),
|
|
"available": int(total) - int(claimed),
|
|
}
|
|
)
|
|
game = {
|
|
"name": name,
|
|
"game_id": num,
|
|
"url": url,
|
|
"price": price,
|
|
"state": "tx",
|
|
"num_tx_initial": num_tx,
|
|
"prizes": prizes,
|
|
}
|
|
return game
|
|
|
|
|
|
def _parse_game(url, html):
|
|
try:
|
|
return parse_game(url, html)
|
|
except Exception as e:
|
|
logger.warning("Unable to parse {}.\n{}".format(url, e))
|
|
return None
|
|
|
|
|
|
def main():
|
|
index_html = fetch_html(INDEX_URL)
|
|
game_urls = parse_index(index_html)
|
|
url_htmls = zip(game_urls, [fetch_html(url) for url in game_urls])
|
|
games = [_parse_game(url, html) for url, html in url_htmls]
|
|
games = [game for game in games if game is not None]
|
|
return games
|
|
|
|
|
|
if __name__ == "__main__":
|
|
games = main()
|
|
schema = GameSchema(many=True)
|
|
print(schema.dumps(games))
|