Johns Hopkins COVID-19 Dataset in Pandas

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from urllib.error import HTTPError
from typing import Optional

plt.style.use("ggplot")

def valid_name(name: str) -> bool:
    "Return True if this is a valid name in the dataset. Faster parsing this way."

    return name.replace("/", "_").replace(" ", "_") in {
        "Country_Region",
        "Province_State",
        "Admin2",
        "Confirmed",
        "Deaths",
        "Recovered",
    }

def get_day(day: pd.Timestamp):
    # Read in a datafile from GitHub
    try:
        table = pd.read_csv(
            "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/"
            "master/csse_covid_19_data/csse_covid_19_daily_reports/"
            f"{day:%m-%d-%Y}.csv",
            usecols=valid_name,
        )
    except HTTPError:
        return pd.DataFrame()

    # Cleanup - sadly, the format has changed a bit over time - we can normalize that here
    table.columns = [f.replace("/", "_").replace(" ", "_") for f in table.columns]

    # This column is new in recent datasets
    if "Admin2" not in table.columns:
        table["Admin2"] = None

    # If the last update time was useful, we would make this day only, rather than day + time
    #   table["Last_Update"] = pd.to_datetime(table["Last_Update"]).dt.normalize()
    #
    # However, last update is odd, let's just make this the current day
    table["Last_Update"] = day

    # Make sure indexes are not NaN, which causes later bits to not work. 0 isn't
    # perfect, but good enough.
    # Return as a multindex
    return table.fillna(0).set_index(
        ["Last_Update", "Country_Region", "Province_State", "Admin2"], drop=True
    )

def get_all_days(end_day: Optional[pd.Timestamp] = None) -> pd.DataFrame:
    # Assume current day - 1 is the latest dataset if no end given
    if end_day is None:
        end_day = pd.Timestamp.now().normalize()

    # Make a list of all dates
    date_range = pd.date_range("2020-01-22", end_day)

    # Create a generator that returns each day's dataframe
    day_gen = (get_day(day) for day in date_range)

    # Make a big dataframe, NaN is 0
    df = pd.concat(day_gen).fillna(0).astype(int)

    # Remove a few duplicate keys
    df = df.groupby(level=df.index.names).sum()

    # Sometimes active is not filled in; we can compute easily
    df["Active"] = np.clip(df["Confirmed"] - df["Deaths"] - df["Recovered"], 0, None)

    # Change in confirmed cases (placed in a pleasing location in the table)
    df.insert(
        1,
        "ΔConfirmed",
        df.groupby(level=("Country_Region", "Province_State", "Admin2"))["Confirmed"]
        .diff()
        .fillna(0)
        .astype(int),
    )

    # Change in deaths
    df.insert(
        3,
        "ΔDeaths",
        df.groupby(level=("Country_Region", "Province_State", "Admin2"))["Deaths"]
        .diff()
        .fillna(0)
        .astype(int),
    )

    return df

df = get_all_days()
df

us = df.xs("US", level="Country_Region")
us

by_state = us.groupby(level=("Last_Update", "Province_State")).sum()
by_state

by_state.xs("North Carolina", level="Province_State")

all_states = by_state.groupby(level="Last_Update").sum()
all_states

plt.figure(figsize=(10, 5))
all_states.Confirmed.plot(logy=True, style="o");

interesting = df.xs("Italy", level="Country_Region").groupby(level="Last_Update").sum()

plt.figure(figsize=(10, 5))
interesting.ΔConfirmed.rolling(5, center=True).mean().plot(
    style="-", label="Rolling mean"
)
interesting.ΔConfirmed.plot(style="o", label="Data")
plt.ylabel("New cases per day")
plt.legend();

interesting = df.xs("Italy", level="Country_Region").groupby(level="Last_Update").sum()
growth = interesting.ΔConfirmed / interesting.Active
growth = growth["2020-02-24":]

# Color based on lockdown (which happened in 3 stages, 4th, 9th, and 11th)
lockdown = growth.index - pd.Timestamp("2020-03-11")
lockdown = np.clip(lockdown.days, 0, 14) / 14

fix, ax = plt.subplots(figsize=(10, 5))
ax.scatter(growth.index, growth, cmap="cool", c=lockdown)

ax.set_ylabel("new cases / active cases")

interesting = df.xs("US", level="Country_Region").groupby(level="Last_Update").sum()
growth = interesting.ΔConfirmed / interesting.Active
growth = growth["2020-03-01":]

# Not really a full lockdown, just a distancing guideline + local lockdowns later
lockdown = growth.index - pd.Timestamp("2020-03-15")
lockdown = np.clip(lockdown.days, 0, 14) / 14

fix, ax = plt.subplots(figsize=(10, 5))
ax.scatter(growth.index, growth, cmap="cool", c=lockdown)

ax.set_ylabel("new cases / active cases")