pbp_com/utils/data_generator.py

import sys, os
sys.path.append('/var/lib/django')

from django.core.management import setup_environ
from pbp_com import settings

setup_environ(settings)


import re
import urllib
import urllib2
from BeautifulSoup import BeautifulSoup
import time
from datetime import datetime, timedelta
import dateutil

from almanac.models import Day, AstroChart

year = '2014'
place = 'Stonington'
state = 'ME'

def get_table(year="", state="", place="", type=""):
    parser = re.compile('\d\d  ')
    '''A special parser for lunar fraction tables '''
    lparser = re.compile(' \d\d  ')

    if type=="sun":
	type=0
    elif type=="moon":
	type=1
    elif type=="civil_tw":
        type=2
    elif type=="nautical_tw":
        type=3
    elif type=="astro_tw":
        type=4 
    elif type=="lunar_fraction": 
        type=5
    else:
	type=0
    
    if type < 5:
        url = "http://aa.usno.navy.mil/cgi-bin/aa_rstablew.pl"
        post_data = {'FFX' : '1',
               'xxy' : year,
               'type' : type,
               'st' : state,
               'place' : place,
               'ZZZ' : 'END'  }
    else:
        url = "http://aa.usno.navy.mil/cgi-bin/aa_moonill2.pl"
        post_data = {'time':'00',
                     'xxy':year,
                     'zone':'-05',
                     'ZZZ':'END'}
                      

    data = urllib.urlencode(post_data)
    req = urllib2.Request(url, data)
    response = urllib2.urlopen(req)
    page = response.read()
    data = page.split('\n')
    
    table=[]
    for line in data:
        if type <5:
            if parser.match(line):
                table.append(line)
        else:
            if lparser.match(line):
                table.append(line) 

    return table
''' End of get_table() '''

def fix_time(str, today, chart):
    if str == '    ': 
        str = None
    else:
        if chart.dst_start < today < chart.dst_end:
            delta=timedelta(hours=1)
        else:
            delta=timedelta(hours=0)
        str= time.strftime("%H:%M", (datetime(*time.strptime(str, "%H%M")[0:5])+delta).timetuple())
    return str     

''' End of fix_time() ''' 


'''### THE HEAVY LIFTING ### '''

sun_table = get_table(year, state, place, 'sun')
moon_table = get_table(year, state, place, 'moon')
civil_tw_table = get_table(year, state, place, 'civil_tw')
naut_tw_table = get_table(year, state, place, 'nautical_tw')
astro_tw_table = get_table(year, state, place, 'astro_tw')
lunar_fraction = get_table(year, '', '', 'lunar_fraction')

month=0
astro_table=[]

''' Here we go over the table 12 times for each month.
    For each month, we keep running till we hit a blank.
    Load the day, month and year into the table and go. '''

offset=4 
moon_iter=0
civil_iter=0
naut_iter=0  
astro_iter=0

lunar_offset=8
lunar_iter=0

try:
    chart=AstroChart.objects.get(year__exact=year)
except AstroChart.DoesNotExist:
    pass #Add code here to add chart if it doesn't exist 


while (month < 12): 
    month = month +1 
    for entry in sun_table:
        if entry[offset:offset+9] == '         ':
            pass
        else:
	    if month < 10:
	        date = datetime.strptime(year + '0' + str(month)+entry[:2], "%Y%m%d").date()
            else: 	
	        date = datetime.strptime(year + str(month) + entry[:2], "%Y%m%d").date()
            astro_table.append({'date':date, 'sunrise':fix_time(entry[offset:offset+4], date,chart), 'sunset':fix_time(entry[offset+5:offset+9], date,chart)}) 

    for entry in moon_table:
        if entry[offset:offset+9] == '         ':
            pass
        else:
            date=astro_table[moon_iter]['date']
            astro_table[moon_iter]['moonrise']=fix_time(entry[offset:offset+4],date, chart) 
            astro_table[moon_iter]['moonset']=fix_time(entry[offset+5:offset+9], date, chart) 
            moon_iter = moon_iter+1

    for entry in civil_tw_table:
        if entry[offset:offset+9] == '         ':
            pass
        else:
            date=astro_table[civil_iter]['date']
            astro_table[civil_iter]['civil_twilight_am']=fix_time(entry[offset:offset+4], date, chart)
            astro_table[civil_iter]['civil_twilight_pm']=fix_time(entry[offset+5:offset+9], date, chart) 
            civil_iter = civil_iter+1

    for entry in naut_tw_table:
        if entry[offset:offset+9] == '         ':
            pass
        else:
            date=astro_table[naut_iter]['date']
            astro_table[naut_iter]['nautical_twilight_am']=fix_time(entry[offset:offset+4], date, chart)
            astro_table[naut_iter]['nautical_twilight_pm']=fix_time(entry[offset+5:offset+9], date, chart) 
            naut_iter = naut_iter+1


    '''
      Want astronomical twilight?
    for entry in astro_tw_table:
        if entry[offset:offset+9] == '         ':
            pass
        else:
            astro_table[astro_iter]['astro_tw_start']=entry[offset:offset+4]
            astro_table[astro_iter]['astro_tw_end']=entry[offset+5:offset+9] 
            astro_iter = astro_iter+1
    '''
    for entry in lunar_fraction:
        if entry[lunar_offset:lunar_offset+4] == '    ':
            pass
        else:
	    frac = entry[lunar_offset:lunar_offset+4]
	    if frac == "1.00":
                astro_table[lunar_iter]['moonphase']=2 # FULL
	    elif frac == "0.00":
                astro_table[lunar_iter]['moonphase']=0 # NEW
	    elif 0.6 > float(frac) > 0.5 and past_phase == "full":
                astro_table[lunar_iter]['moonphase']=3 # 3rd Q.
	    elif 0.6 > float(frac) > 0.5 and past_phase == "new":
                astro_table[lunar_iter]['moonphase']=1 # 1st Q.

            astro_table[lunar_iter]['moonphase_fraction']=entry[lunar_offset:lunar_offset+4]
            lunar_iter = lunar_iter+1
        if frac == "1.00":
            past_phase = "full"
        elif frac=="0.00":  
            past_phase = "new"

    lunar_offset=lunar_offset +9
    offset=offset +11

''' Load the data into the DB '''


for entry in astro_table:
    print entry
    entry['chart']=chart
    day=Day.objects.create(**entry)
    day.save()