Source code for writer.storage

""" Storage docstrings
"""
import os

import tables


class HisparcEvent(tables.IsDescription):
    # DISCUSS: use of signed (dflt -1) vs unsigned (labview code)
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    nanoseconds = tables.UInt32Col(pos=2)
    ext_timestamp = tables.UInt64Col(pos=3)
    data_reduction = tables.BoolCol(pos=4)
    trigger_pattern = tables.UInt32Col(pos=5)
    baseline = tables.Int16Col(shape=4, dflt=-1, pos=6)
    std_dev = tables.Int16Col(shape=4, dflt=-1, pos=7)
    n_peaks = tables.Int16Col(shape=4, dflt=-1, pos=8)
    pulseheights = tables.Int16Col(shape=4, dflt=-1, pos=9)
    integrals = tables.Int32Col(shape=4, dflt=-1, pos=10)
    traces = tables.Int32Col(shape=4, dflt=-1, pos=11)
    event_rate = tables.Float32Col(pos=12)


class HisparcError(tables.IsDescription):
    """HiSPARC Error messages tables"""
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    messages = tables.Int32Col(dflt=-1, pos=2)


class HisparcConfiguration(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    gps_latitude = tables.Float64Col(pos=2)
    gps_longitude = tables.Float64Col(pos=3)
    gps_altitude = tables.Float64Col(pos=4)
    mas_version = tables.Int32Col(dflt=-1, pos=5)
    slv_version = tables.Int32Col(dflt=-1, pos=6)
    trig_low_signals = tables.UInt32Col(pos=7)
    trig_high_signals = tables.UInt32Col(pos=8)
    trig_external = tables.UInt32Col(pos=9)
    trig_and_or = tables.BoolCol(pos=10)
    precoinctime = tables.Float64Col(pos=11)
    coinctime = tables.Float64Col(pos=12)
    postcoinctime = tables.Float64Col(pos=13)
    detnum = tables.UInt16Col(pos=14)
    password = tables.Int32Col(dflt=-1, pos=15)
    spare_bytes = tables.UInt8Col(pos=16)
    use_filter = tables.BoolCol(pos=17)
    use_filter_threshold = tables.BoolCol(pos=18)
    reduce_data = tables.BoolCol(pos=19)
    buffer = tables.Int32Col(dflt=-1, pos=20)
    startmode = tables.BoolCol(pos=21)
    delay_screen = tables.Float64Col(pos=22)
    delay_check = tables.Float64Col(pos=23)
    delay_error = tables.Float64Col(pos=24)
    mas_ch1_thres_low = tables.Float64Col(pos=25)
    mas_ch1_thres_high = tables.Float64Col(pos=26)
    mas_ch2_thres_low = tables.Float64Col(pos=27)
    mas_ch2_thres_high = tables.Float64Col(pos=28)
    mas_ch1_inttime = tables.Float64Col(pos=29)
    mas_ch2_inttime = tables.Float64Col(pos=30)
    mas_ch1_voltage = tables.Float64Col(pos=31)
    mas_ch2_voltage = tables.Float64Col(pos=32)
    mas_ch1_current = tables.Float64Col(pos=33)
    mas_ch2_current = tables.Float64Col(pos=34)
    mas_comp_thres_low = tables.Float64Col(pos=35)
    mas_comp_thres_high = tables.Float64Col(pos=36)
    mas_max_voltage = tables.Float64Col(pos=37)
    mas_reset = tables.BoolCol(pos=38)
    mas_ch1_gain_pos = tables.UInt8Col(pos=39)
    mas_ch1_gain_neg = tables.UInt8Col(pos=40)
    mas_ch2_gain_pos = tables.UInt8Col(pos=41)
    mas_ch2_gain_neg = tables.UInt8Col(pos=42)
    mas_ch1_offset_pos = tables.UInt8Col(pos=43)
    mas_ch1_offset_neg = tables.UInt8Col(pos=44)
    mas_ch2_offset_pos = tables.UInt8Col(pos=45)
    mas_ch2_offset_neg = tables.UInt8Col(pos=46)
    mas_common_offset = tables.UInt8Col(pos=47)
    mas_internal_voltage = tables.UInt8Col(pos=48)
    mas_ch1_adc_gain = tables.Float64Col(pos=49)
    mas_ch1_adc_offset = tables.Float64Col(pos=50)
    mas_ch2_adc_gain = tables.Float64Col(pos=51)
    mas_ch2_adc_offset = tables.Float64Col(pos=52)
    mas_ch1_comp_gain = tables.Float64Col(pos=53)
    mas_ch1_comp_offset = tables.Float64Col(pos=54)
    mas_ch2_comp_gain = tables.Float64Col(pos=55)
    mas_ch2_comp_offset = tables.Float64Col(pos=56)
    slv_ch1_thres_low = tables.Float64Col(pos=57)
    slv_ch1_thres_high = tables.Float64Col(pos=58)
    slv_ch2_thres_low = tables.Float64Col(pos=59)
    slv_ch2_thres_high = tables.Float64Col(pos=60)
    slv_ch1_inttime = tables.Float64Col(pos=61)
    slv_ch2_inttime = tables.Float64Col(pos=62)
    slv_ch1_voltage = tables.Float64Col(pos=63)
    slv_ch2_voltage = tables.Float64Col(pos=64)
    slv_ch1_current = tables.Float64Col(pos=65)
    slv_ch2_current = tables.Float64Col(pos=66)
    slv_comp_thres_low = tables.Float64Col(pos=67)
    slv_comp_thres_high = tables.Float64Col(pos=68)
    slv_max_voltage = tables.Float64Col(pos=69)
    slv_reset = tables.BoolCol(pos=70)
    slv_ch1_gain_pos = tables.UInt8Col(pos=71)
    slv_ch1_gain_neg = tables.UInt8Col(pos=72)
    slv_ch2_gain_pos = tables.UInt8Col(pos=73)
    slv_ch2_gain_neg = tables.UInt8Col(pos=74)
    slv_ch1_offset_pos = tables.UInt8Col(pos=75)
    slv_ch1_offset_neg = tables.UInt8Col(pos=76)
    slv_ch2_offset_pos = tables.UInt8Col(pos=77)
    slv_ch2_offset_neg = tables.UInt8Col(pos=78)
    slv_common_offset = tables.UInt8Col(pos=79)
    slv_internal_voltage = tables.UInt8Col(pos=80)
    slv_ch1_adc_gain = tables.Float64Col(pos=81)
    slv_ch1_adc_offset = tables.Float64Col(pos=82)
    slv_ch2_adc_gain = tables.Float64Col(pos=83)
    slv_ch2_adc_offset = tables.Float64Col(pos=84)
    slv_ch1_comp_gain = tables.Float64Col(pos=85)
    slv_ch1_comp_offset = tables.Float64Col(pos=86)
    slv_ch2_comp_gain = tables.Float64Col(pos=87)
    slv_ch2_comp_offset = tables.Float64Col(pos=88)


class HisparcComparator(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    nanoseconds = tables.UInt32Col(pos=2)
    ext_timestamp = tables.UInt64Col(pos=3)
    device = tables.UInt8Col(pos=4)
    comparator = tables.UInt8Col(pos=5)
    count = tables.UInt16Col(pos=6)


class HisparcSingle(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    mas_ch1_low = tables.Int32Col(dflt=-1, pos=2)
    mas_ch1_high = tables.Int32Col(dflt=-1, pos=3)
    mas_ch2_low = tables.Int32Col(dflt=-1, pos=4)
    mas_ch2_high = tables.Int32Col(dflt=-1, pos=5)
    slv_ch1_low = tables.Int32Col(dflt=-1, pos=6)
    slv_ch1_high = tables.Int32Col(dflt=-1, pos=7)
    slv_ch2_low = tables.Int32Col(dflt=-1, pos=8)
    slv_ch2_high = tables.Int32Col(dflt=-1, pos=9)


class HisparcSatellite(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    min_n = tables.UInt16Col(pos=2)
    mean_n = tables.Float32Col(pos=3)
    max_n = tables.UInt16Col(pos=4)
    min_signal = tables.UInt16Col(pos=5)
    mean_signal = tables.Float32Col(pos=6)
    max_signal = tables.UInt16Col(pos=7)


class WeatherEvent(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    temp_inside = tables.Float32Col(pos=2)
    temp_outside = tables.Float32Col(pos=3)
    humidity_inside = tables.Int16Col(pos=4)
    humidity_outside = tables.Int16Col(pos=5)
    barometer = tables.Float32Col(pos=6)
    wind_dir = tables.Int16Col(pos=7)
    wind_speed = tables.Int16Col(pos=8)
    solar_rad = tables.Int16Col(pos=9)
    uv = tables.Int16Col(pos=10)
    evapotranspiration = tables.Float32Col(pos=11)
    rain_rate = tables.Float32Col(pos=12)
    heat_index = tables.Int16Col(pos=13)
    dew_point = tables.Float32Col(pos=14)
    wind_chill = tables.Float32Col(pos=15)


class WeatherError(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    messages = tables.Int32Col(dflt=-1, pos=2)


class WeatherConfig(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    com_port = tables.UInt8Col(pos=2)
    baud_rate = tables.Int16Col(pos=3)
    station_id = tables.UInt32Col(pos=4)
    database_name = tables.Int32Col(dflt=-1, pos=5)
    help_url = tables.Int32Col(dflt=-1, pos=6)
    daq_mode = tables.BoolCol(pos=7)
    latitude = tables.Float64Col(pos=8)
    longitude = tables.Float64Col(pos=9)
    altitude = tables.Float64Col(pos=10)
    temperature_inside = tables.BoolCol(pos=11)
    temperature_outside = tables.BoolCol(pos=12)
    humidity_inside = tables.BoolCol(pos=13)
    humidity_outside = tables.BoolCol(pos=14)
    barometer = tables.BoolCol(pos=15)
    wind_direction = tables.BoolCol(pos=16)
    wind_speed = tables.BoolCol(pos=17)
    solar_radiation = tables.BoolCol(pos=18)
    uv_index = tables.BoolCol(pos=19)
    evapotranspiration = tables.BoolCol(pos=20)
    rain_rate = tables.BoolCol(pos=21)
    heat_index = tables.BoolCol(pos=22)
    dew_point = tables.BoolCol(pos=23)
    wind_chill = tables.BoolCol(pos=24)
    offset_inside_temperature = tables.Float32Col(pos=25)
    offset_outside_temperature = tables.Float32Col(pos=26)
    offset_inside_humidity = tables.Int16Col(pos=27)
    offset_outside_humidity = tables.Int16Col(pos=28)
    offset_wind_direction = tables.Int16Col(pos=29)
    offset_station_altitude = tables.Float32Col(pos=30)
    offset_bar_sea_level = tables.Float32Col(pos=31)


class LightningEvent(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    corr_distance = tables.Int16Col(pos=2)
    uncorr_distance = tables.Int16Col(pos=3)
    uncorr_angle = tables.Float32Col(pos=4)
    corr_angle = tables.Float32Col(pos=5)


class LightningError(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    messages = tables.Int32Col(dflt=-1, pos=2)


class LightningConfig(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    com_port = tables.UInt8Col(pos=2)
    baud_rate = tables.Int16Col(pos=3)
    station_id = tables.UInt32Col(pos=4)
    database_name = tables.Int32Col(dflt=-1, pos=5)
    help_url = tables.Int32Col(dflt=-1, pos=6)
    daq_mode = tables.BoolCol(pos=7)
    latitude = tables.Float64Col(pos=8)
    longitude = tables.Float64Col(pos=9)
    altitude = tables.Float64Col(pos=10)
    squelch_seting = tables.Int32Col(pos=11)
    close_alarm_distance = tables.Int32Col(pos=12)
    severe_alarm_distance = tables.Int32Col(pos=13)
    noise_beep = tables.BoolCol(pos=14)
    minimum_gps_speed = tables.Int32Col(pos=15)
    angle_correction = tables.Float32Col(pos=16)


class LightningStatus(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)
    close_rate = tables.Int16Col(pos=2)
    total_rate = tables.Int16Col(pos=3)
    close_alarm = tables.BoolCol(pos=4)
    sever_alarm = tables.BoolCol(pos=5)
    current_heading = tables.Float32Col(pos=6)


class LightningNoise(tables.IsDescription):
    event_id = tables.UInt32Col(pos=0)
    timestamp = tables.Time32Col(pos=1)


[docs]def open_or_create_file(data_dir, date): """Open an existing file or create a new one This function opens an existing PyTables file according to the event date. If the file does not yet exist, a new one is created. :param data_dir: the directory containing all data files :param date: the event date """ dir = os.path.join(data_dir, '%d/%d' % (date.year, date.month)) file = os.path.join(dir, '%d_%d_%d.h5' % (date.year, date.month, date.day)) if not os.path.exists(dir): # create dir and parent dirs with mode rwxr-xr-x os.makedirs(dir, 0o755) return tables.open_file(file, 'a')
[docs]def get_or_create_station_group(file, cluster, station_id): """Get an existing station group or create a new one :param file: the PyTables data file :param cluster: the name of the cluster :param station_id: the station number """ cluster = get_or_create_cluster_group(file, cluster) node_name = 'station_%d' % station_id try: station = file.get_node(cluster, node_name) except tables.NoSuchNodeError: station = file.create_group(cluster, node_name, 'HiSPARC station %d data' % station_id) file.flush() return station
[docs]def get_or_create_cluster_group(file, cluster): """Get an existing cluster group or create a new one :param file: the PyTables data file :param cluster: the name of the cluster """ try: hisparc = file.get_node('/', 'hisparc') except tables.NoSuchNodeError: hisparc = file.create_group('/', 'hisparc', 'HiSPARC data') file.flush() node_name = 'cluster_' + cluster.lower() try: cluster = file.get_node(hisparc, node_name) except tables.NoSuchNodeError: cluster = file.create_group(hisparc, node_name, 'HiSPARC cluster %s data' % cluster) file.flush() return cluster
[docs]def get_or_create_node(file, cluster, node): """Get an existing node or create a new one :param file: the PyTables data file :param cluster: the parent (cluster) node :param node: the node (e.g. events, blobs) """ try: node = file.get_node(cluster, node) except tables.NoSuchNodeError: if node == 'events': node = file.create_table(cluster, 'events', HisparcEvent, 'HiSPARC event data') elif node == 'errors': node = file.create_table(cluster, 'errors', HisparcError, 'HiSPARC error messages') elif node == 'config': node = file.create_table(cluster, 'config', HisparcConfiguration, 'HiSPARC configuration messages') elif node == 'comparator': node = file.create_table(cluster, 'comparator', HisparcComparator, 'HiSPARC comparator messages') elif node == 'singles': node = file.create_table(cluster, 'singles', HisparcSingle, 'HiSPARC single messages') elif node == 'satellites': node = file.create_table(cluster, 'satellites', HisparcSatellite, 'HiSPARC satellite messages') elif node == 'blobs': node = file.create_vlarray(cluster, 'blobs', tables.VLStringAtom(), 'HiSPARC binary data') elif node == 'weather': node = file.create_table(cluster, 'weather', WeatherEvent, 'HiSPARC weather data') elif node == 'weather_errors': node = file.create_table(cluster, 'weather_errors', WeatherError, 'HiSPARC weather error messages') elif node == 'weather_config': node = file.create_table(cluster, 'weather_config', WeatherConfig, 'HiSPARC weather configuration messages') elif node == 'lightning': node = file.create_table(cluster, 'lightning', LightningEvent, 'HiSPARC lightning data') elif node == 'lightning_errors': node = file.create_table(cluster, 'lightning_errors', LightningError, 'HiSPARC lightning error messages') elif node == 'lightning_config': node = file.create_table(cluster, 'lightning_config', LightningConfig, 'HiSPARC lightning configuration ' 'messages') elif node == 'lightning_status': node = file.create_table(cluster, 'lightning_status', LightningStatus, 'HiSPARC lightning status messages') elif node == 'lightning_noise': node = file.create_table(cluster, 'lightning_noise', LightningNoise, 'HiSPARC lightning noise messages') file.flush() return node