pbp_com/utils/data_generator.py

import sys, os

sys.path.append("/var/lib/django")

from django.core.management import setup_environ
from pbp_com import settings

setup_environ(settings)


import re
import urllib
import urllib2
from BeautifulSoup import BeautifulSoup
import time
from datetime import datetime, timedelta
import dateutil

from almanac.models import Day, AstroChart

year = "2014"
place = "Stonington"
state = "ME"


def get_table(year="", state="", place="", type=""):
    parser = re.compile("\d\d  ")
    """A special parser for lunar fraction tables """
    lparser = re.compile(" \d\d  ")

    if type == "sun":
        type = 0
    elif type == "moon":
        type = 1
    elif type == "civil_tw":
        type = 2
    elif type == "nautical_tw":
        type = 3
    elif type == "astro_tw":
        type = 4
    elif type == "lunar_fraction":
        type = 5
    else:
        type = 0

    if type < 5:
        url = "http://aa.usno.navy.mil/cgi-bin/aa_rstablew.pl"
        post_data = {
            "FFX": "1",
            "xxy": year,
            "type": type,
            "st": state,
            "place": place,
            "ZZZ": "END",
        }
    else:
        url = "http://aa.usno.navy.mil/cgi-bin/aa_moonill2.pl"
        post_data = {"time": "00", "xxy": year, "zone": "-05", "ZZZ": "END"}

    data = urllib.urlencode(post_data)
    req = urllib2.Request(url, data)
    response = urllib2.urlopen(req)
    page = response.read()
    data = page.split("\n")

    table = []
    for line in data:
        if type < 5:
            if parser.match(line):
                table.append(line)
        else:
            if lparser.match(line):
                table.append(line)

    return table


""" End of get_table() """


def fix_time(str, today, chart):
    if str == "    ":
        str = None
    else:
        if chart.dst_start < today < chart.dst_end:
            delta = timedelta(hours=1)
        else:
            delta = timedelta(hours=0)
        str = time.strftime(
            "%H:%M", (datetime(*time.strptime(str, "%H%M")[0:5]) + delta).timetuple()
        )
    return str


""" End of fix_time() """


"""### THE HEAVY LIFTING ### """

sun_table = get_table(year, state, place, "sun")
moon_table = get_table(year, state, place, "moon")
civil_tw_table = get_table(year, state, place, "civil_tw")
naut_tw_table = get_table(year, state, place, "nautical_tw")
astro_tw_table = get_table(year, state, place, "astro_tw")
lunar_fraction = get_table(year, "", "", "lunar_fraction")

month = 0
astro_table = []

""" Here we go over the table 12 times for each month.
    For each month, we keep running till we hit a blank.
    Load the day, month and year into the table and go. """

offset = 4
moon_iter = 0
civil_iter = 0
naut_iter = 0
astro_iter = 0

lunar_offset = 8
lunar_iter = 0

try:
    chart = AstroChart.objects.get(year__exact=year)
except AstroChart.DoesNotExist:
    pass  # Add code here to add chart if it doesn't exist


while month < 12:
    month = month + 1
    for entry in sun_table:
        if entry[offset : offset + 9] == "         ":
            pass
        else:
            if month < 10:
                date = datetime.strptime(
                    year + "0" + str(month) + entry[:2], "%Y%m%d"
                ).date()
            else:
                date = datetime.strptime(year + str(month) + entry[:2], "%Y%m%d").date()
            astro_table.append(
                {
                    "date": date,
                    "sunrise": fix_time(entry[offset : offset + 4], date, chart),
                    "sunset": fix_time(entry[offset + 5 : offset + 9], date, chart),
                }
            )

    for entry in moon_table:
        if entry[offset : offset + 9] == "         ":
            pass
        else:
            date = astro_table[moon_iter]["date"]
            astro_table[moon_iter]["moonrise"] = fix_time(
                entry[offset : offset + 4], date, chart
            )
            astro_table[moon_iter]["moonset"] = fix_time(
                entry[offset + 5 : offset + 9], date, chart
            )
            moon_iter = moon_iter + 1

    for entry in civil_tw_table:
        if entry[offset : offset + 9] == "         ":
            pass
        else:
            date = astro_table[civil_iter]["date"]
            astro_table[civil_iter]["civil_twilight_am"] = fix_time(
                entry[offset : offset + 4], date, chart
            )
            astro_table[civil_iter]["civil_twilight_pm"] = fix_time(
                entry[offset + 5 : offset + 9], date, chart
            )
            civil_iter = civil_iter + 1

    for entry in naut_tw_table:
        if entry[offset : offset + 9] == "         ":
            pass
        else:
            date = astro_table[naut_iter]["date"]
            astro_table[naut_iter]["nautical_twilight_am"] = fix_time(
                entry[offset : offset + 4], date, chart
            )
            astro_table[naut_iter]["nautical_twilight_pm"] = fix_time(
                entry[offset + 5 : offset + 9], date, chart
            )
            naut_iter = naut_iter + 1

    """
      Want astronomical twilight?
    for entry in astro_tw_table:
        if entry[offset:offset+9] == '         ':
            pass
        else:
            astro_table[astro_iter]['astro_tw_start']=entry[offset:offset+4]
            astro_table[astro_iter]['astro_tw_end']=entry[offset+5:offset+9] 
            astro_iter = astro_iter+1
    """
    for entry in lunar_fraction:
        if entry[lunar_offset : lunar_offset + 4] == "    ":
            pass
        else:
            frac = entry[lunar_offset : lunar_offset + 4]
            if frac == "1.00":
                astro_table[lunar_iter]["moonphase"] = 2  # FULL
            elif frac == "0.00":
                astro_table[lunar_iter]["moonphase"] = 0  # NEW
            elif 0.6 > float(frac) > 0.5 and past_phase == "full":
                astro_table[lunar_iter]["moonphase"] = 3  # 3rd Q.
            elif 0.6 > float(frac) > 0.5 and past_phase == "new":
                astro_table[lunar_iter]["moonphase"] = 1  # 1st Q.

            astro_table[lunar_iter]["moonphase_fraction"] = entry[
                lunar_offset : lunar_offset + 4
            ]
            lunar_iter = lunar_iter + 1
        if frac == "1.00":
            past_phase = "full"
        elif frac == "0.00":
            past_phase = "new"

    lunar_offset = lunar_offset + 9
    offset = offset + 11

""" Load the data into the DB """


for entry in astro_table:
    print entry
    entry["chart"] = chart
    day = Day.objects.create(**entry)
    day.save()