IO & Data Access

This module takes care of receiving the data catalogs.

source

postprocessor


def postprocessor(
    fname, # Full path of the zipped file in local storage
    action, # One of "download" (file doesn't exist and will download),
"update" (file is outdated and will download), and
"fetch" (file exists and is updated so no download).
    pup, # The instance of Pooch that called the processor function.
): # The full path to the unzipped file. (Return the same fname is your
processor doesn't modify the file).

Post-processing hook to process the downloaded v1 catalog file.

Postprocessing steps are: 1. unzip to CSV 2. load CSV into pandas 3. save as parquet file.

This time, at the next read, the performance is much increased due to reading the parquet file instead of loading the CSV file every time.

v1.fetch("fans", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v1/fans.unzip/P4_catalog_v1.1_L1C_cut_0.5_fan.parq')
v1.fetch("blotches", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v1/blotches.unzip/P4_catalog_v1.1_L1C_cut_0.5_blotch.parq')
pd.read_parquet(v1.fetch("metadata", **kwargs))
OBSERVATION_ID IMAGE_CENTER_LATITUDE IMAGE_CENTER_LONGITUDE SOLAR_LONGITUDE START_TIME map_scale north_azimuth # of tiles
0 ESP_011296_0975 -82.1965 225.2530 178.833 2008-12-23 16:15:26 1.00 110.600107 91
1 ESP_011341_0980 -81.7969 76.1304 180.809 2008-12-27 04:25:02 0.50 110.208923 126
2 ESP_011348_0950 -85.0427 259.0940 181.117 2008-12-27 17:29:17 1.00 123.624057 91
3 ESP_011350_0945 -85.2160 181.4150 181.205 2008-12-27 21:14:01 0.50 99.672793 126
4 ESP_011351_0945 -85.2157 181.5480 181.249 2008-12-27 23:05:54 1.00 127.960688 91
... ... ... ... ... ... ... ... ...
216 ESP_022273_0950 -84.8184 65.7963 282.247 2011-04-28 00:21:01 0.25 117.236648 270
217 ESP_022339_0935 -86.2751 99.6123 285.423 2011-05-03 03:45:55 0.25 153.768164 315
218 ESP_022379_0930 -87.0045 86.5510 287.340 2011-05-06 06:34:07 0.25 153.003708 260
219 ESP_022510_0980 -81.6988 66.3188 293.572 2011-05-16 11:34:43 0.50 107.877998 72
220 ESP_022699_0985 -81.4043 295.7820 302.421 2011-05-31 05:01:48 0.50 96.757068 90

221 rows × 8 columns

v3.fetch("fans", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v3/fans.parq')
v3.fetch("blotches", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v3/blotches.parq')
v3.fetch("tile_coords")
'/Users/maye/Library/Caches/p4tools/v3/tile_coords'

source

catalog_version


def catalog_version(
    version:str, # Catalog version string, e.g. ``'v3.1'``, ``'v3'``, ``'v1'``.
):

Context manager to temporarily use a different catalog version.

source

set_catalog_version


def set_catalog_version(
    version:str, # Catalog version string, e.g. ``'v3.1'``, ``'v3'``, ``'v1'``.
):

Set the default catalog version for the current session/notebook.

source

define_martian_year


def define_martian_year(
    df, # DataFrame to modify
    time_col_name, # Name of the time column to use for MY calculation
): # DataFrame with added 'MY' column

Add Martian Year (MY) column to dataframe based on time column.

source

get_tile_urls


def get_tile_urls(
    
)->DataFrame:

only v1 exists

source

get_region_names


def get_region_names(
    version:NoneType=None
)->DataFrame:

Per-obsid ROI assignments.

v3.1 (default) includes the 11 MY-33 obsids that are missing from the v1 mapping (5 Manhattan_Classic + 6 Ithaca). v1 is kept available for historical reproduction.

source

get_meta_data


def get_meta_data(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_tile_coords


def get_tile_coords(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_fan_catalog


def get_fan_catalog(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_blotch_catalog


def get_blotch_catalog(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_metafull


def get_metafull(
    version:NoneType=None
)->DataFrame:

only v3 and v3.1 exist

fans = get_fan_catalog()
fans["MY"].value_counts().sort_index()
MY
28     85065
29    101328
30     57779
31     15528
32      7481
33      6240
Name: count, dtype: int64
meta = get_metafull()
[col for col in meta.columns if "TIME" in col]
['OBSERVATION_START_TIME',
 'START_TIME',
 'STOP_TIME',
 'DELTA_LINE_TIME_COUNT',
 'LOCAL_TIME']
meta.columns
Index(['OBSERVATION_ID', 'VOLUME_ID', 'FILE_NAME_SPECIFICATION',
       'INSTRUMENT_HOST_ID', 'INSTRUMENT_ID', 'PRODUCT_ID',
       'PRODUCT_VERSION_ID', 'HICAL_VERSION', 'TARGET_NAME', 'ORBIT_NUMBER',
       'MISSION_PHASE_NAME', 'RATIONALE_DESC', 'OBSERVATION_START_TIME',
       'OBSERVATION_START_COUNT', 'START_TIME', 'SPACECRAFT_CLOCK_START_COUNT',
       'STOP_TIME', 'SPACECRAFT_CLOCK_STOP_COUNT', 'CCD_NAME',
       'CHANNEL_NUMBER', 'FILTER_NAME', 'SCAN_EXPOSURE_DURATION',
       'DELTA_LINE_TIME_COUNT', 'BINNING', 'TDI', 'TRIM_LINES',
       'FOCUS_POSITION_COUNT', 'FELICS_COMPRESSION_FLAG',
       'STIMULATION_LAMP_FLAG_RED', 'STIMULATION_LAMP_FLAG_BLUEGREEN',
       'STIMULATION_LAMP_FLAG_NEARINFRARED', 'LOOKUP_TABLE_TYPE',
       'LOOKUP_TABLE_MINIMUM', 'LOOKUP_TABLE_MAXIMUM', 'LOOKUP_TABLE_MEDIAN',
       'LOOKUP_TABLE_K_VALUE', 'LOOKUP_TABLE_NUMBER',
       'ADC_CONVERSION_SETTINGS_1', 'ADC_CONVERSION_SETTINGS_2',
       'FPA_POSITIVE_Y_TEMPERATURE', 'FPA_NEGATIVE_Y_TEMPERATURE',
       'FPE_TEMPERATURE', 'IEA_TEMPERATURE', 'IEA_PWS_BOARD_TEMPERATURE',
       'CPMM_PWS_BOARD_TEMPERATURE', 'IMAGE_LINES', 'LINE_SAMPLES',
       'SAMPLE_BITS', 'SCALED_PIXEL_WIDTH', 'EMISSION_ANGLE',
       'INCIDENCE_ANGLE', 'PHASE_ANGLE', 'IMAGE_CENTER_LATITUDE',
       'IMAGE_CENTER_LONGITUDE', 'MINIMUM_LATITUDE', 'MAXIMUM_LATITUDE',
       'MINIMUM_LONGITUDE', 'MAXIMUM_LONGITUDE', 'SPACECRAFT_ALTITUDE',
       'TARGET_CENTER_DISTANCE', 'SLANT_DISTANCE', 'NORTH_AZIMUTH',
       'SUB_SOLAR_AZIMUTH', 'SUB_SOLAR_LATITUDE', 'SUB_SOLAR_LONGITUDE',
       'SUB_SPACECRAFT_LATITUDE', 'SUB_SPACECRAFT_LONGITUDE', 'SOLAR_DISTANCE',
       'SOLAR_LONGITUDE', 'LOCAL_TIME', 'STEREO_FLAG', 'north_azimuth',
       '# of tiles', 'map_scale', 'MY'],
      dtype='object')
get_blotch_catalog("v1").tail()
obsid marking_id angle tile_id image_x image_y n_votes radius_1 radius_2 vote_ratio ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY
249796 ESP_022699_0985 B044be7 0.00 APF0000idg 1276.20 14257.20 5 10.00 10.0 1.0 ... 302.421 0.5 96.757068 219.906430 -455.912914 -3342.551722 -81.388876 -81.488458 295.750015 30
249797 ESP_022699_0985 B044be8 0.00 APF0000idg 1311.25 14345.75 4 10.00 10.0 1.0 ... 302.421 0.5 96.757068 219.897692 -455.963924 -3342.536671 -81.388131 -81.487722 295.746616 30
249798 ESP_022699_0985 B044be9 0.00 APF0000idg 1266.00 14355.25 4 10.00 10.0 1.0 ... 302.421 0.5 96.757068 219.925196 -455.960274 -3342.537242 -81.387987 -81.487580 295.749599 30
249799 ESP_022699_0985 B044bea 168.67 APF0000idh 74.07 148.86 7 16.84 14.1 1.0 ... 302.421 0.5 96.757068 218.194755 -448.614747 -3343.589608 -81.514080 -81.612257 295.937105 30
249800 ESP_022699_0985 B044beb 172.85 APF0000idh 265.00 98.33 3 40.55 25.6 1.0 ... 302.421 0.5 96.757068 218.086509 -448.624657 -3343.607336 -81.514768 -81.612938 295.925426 30

5 rows × 24 columns

get_region_names()
obsid lat_IND lon_IND roi_name minimal_distance lat_WORD lon_WORD time MY
0 ESP_012079_0945 -85.4074 103.970 Macclesfield 0.944588 -85.401 103.901 2009-02-22 16:32:19 29
1 ESP_021494_0945 -85.4041 103.943 Macclesfield 0.511133 -85.401 103.901 2011-02-26 07:30:27 30
2 ESP_011407_0945 -85.4065 103.983 Macclesfield 0.957066 -85.401 103.901 2009-01-01 07:49:15 29
3 ESP_020782_0945 -85.4071 103.967 Macclesfield 0.901696 -85.401 103.901 2011-01-01 19:57:51 30
4 ESP_020716_0945 -85.4086 103.979 Macclesfield 1.098928 -85.401 103.901 2010-12-27 16:32:24 30
... ... ... ... ... ... ... ... ... ...
449 ESP_029762_0940 -85.7567 106.026 Manhattan2 0.782410 -85.751 105.971 2012-12-01 13:16:26 31
450 ESP_037977_0940 -85.6793 105.614 Manhattan2 8.544184 -85.751 105.971 2014-09-02 16:34:56 32
451 ESP_037976_0940 -85.6694 105.903 Manhattan2 9.131402 -85.751 105.971 2014-09-02 14:43:05 32
452 ESP_040311_0940 -85.7806 106.025 Manhattan2 3.335817 -85.751 105.971 2015-03-03 12:35:10 32
453 ESP_040193_0940 -85.7860 106.143 Manhattan2 4.158145 -85.751 105.971 2015-02-22 07:54:37 32

454 rows × 9 columns

get_tile_urls()
tile_id tile_url
0 APF0000coq http://www.planetfour.org/subjects/standard/50...
1 APF0000cro http://www.planetfour.org/subjects/standard/50...
2 APF0000cs6 http://www.planetfour.org/subjects/standard/50...
3 APF0000co1 http://www.planetfour.org/subjects/standard/50...
4 APF0000cpd http://www.planetfour.org/subjects/standard/50...
... ... ...
114866 APF0002eep https://www.planetfour.org/subjects/standard/5...
114867 APF0002ee9 https://www.planetfour.org/subjects/standard/5...
114868 APF0002eej https://www.planetfour.org/subjects/standard/5...
114869 APF0002ee0 https://www.planetfour.org/subjects/standard/5...
114870 APF0002edl https://www.planetfour.org/subjects/standard/5...

114871 rows × 2 columns

source

normalize_tile_id


def normalize_tile_id(
    tile_id:str, # Full or partial tile ID. If partial, it will be padded with 'APF' and leading zeros.
)->str: # Complete tile ID in format 'APF0000xxx' (always 9 characters)

Normalize a tile ID by adding ‘APF’ prefix and leading zeros if necessary.

assert normalize_tile_id("ci9") == "APF0000ci9"

source

get_subframe


def get_subframe(
    url, # Full subframe image URL.
    download:bool=True, # If False and the image is not already cached, raise instead of
attempting a network download (the origin host is offline).
):

Load a subframe image for url from the local pooch cache.

source

is_tile_cached


def is_tile_cached(
    tile_id
):

True if the subframe image for tile_id is already in the local cache.

source

get_url_for_tile


def get_url_for_tile(
    tile_id
):

Deprecated alias for get_url_for_tile_id.

source

get_url_for_tile_id


def get_url_for_tile_id(
    tile_id
):

Return the subframe image URL for tile_id.

tile_id = "APF0000ci9"
tile_id = "1xbr"
get_url_for_tile(tile_id)
'http://www.planetfour.org/subjects/standard/5143480fea305267e900b75e.jpg'
get_url_for_tile_id("ci9")
'http://www.planetfour.org/subjects/standard/50e7429e5e2ed21240003f51.jpg'

source

get_subframe_for_tile


def get_subframe_for_tile(
    tile_id
):

Deprecated alias for get_subframe_by_tile_id.

source

get_subframe_by_tile_id


def get_subframe_by_tile_id(
    tile_id, download:bool=True
):

Load the subframe image for tile_id from the local cache.

source

get_cached_tile_ids


def get_cached_tile_ids(
    
): # Sorted tile_ids whose subframe image is present in the local cache.

Reverse-map the local subframe cache to the tile_ids available offline.

Scans the pooch tiles cache (pooch.os_cache("p4tools/tiles")), where each file is named <md5(url)>-<subject>.jpg, and maps it back to its tile_id via get_tile_urls. Only files whose name matches pooch.utils.unique_file_name of the looked-up URL are counted, so stray files are ignored.

plt.imshow(get_subframe_for_tile(tile_id))

source

get_fans_for_tile


def get_fans_for_tile(
    tile_id, version:NoneType=None
):

Call self as a function.

get_fans_for_tile("1xbr").sort_values(by="angle")
obsid marking_id angle distance tile_id image_x image_y n_votes spread version ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY
201216 PSP_002942_0935 F031200 36.81 798.87 APF0001xbr 31.62 28633.00 4.0 25.29 1 ... 199.642 0.5 135.831931 -30.228735 215.484939 -3374.391494 -86.310437 -86.353625 97.985477 28
201217 PSP_002942_0935 F031201 37.85 776.30 APF0001xbr 50.80 28635.33 3.0 14.58 1 ... 199.642 0.5 135.831931 -30.220416 215.480259 -3374.391442 -86.310535 -86.353722 97.983479 28
201215 PSP_002942_0935 F0311ff 40.44 597.17 APF0001xbr 64.76 28626.95 6.0 22.23 1 ... 199.642 0.5 135.831931 -30.217288 215.472891 -3374.391918 -86.310666 -86.353851 97.982932 28
201214 PSP_002942_0935 F0311fe 40.83 314.90 APF0001xbr 275.16 28732.00 3.0 18.79 1 ... 199.642 0.5 135.831931 -30.103163 215.453344 -3374.393449 -86.311262 -86.354441 97.953881 28

4 rows × 25 columns

get_fans_for_tile("1xbr", version="v1").sort_values(by="angle")
obsid marking_id angle distance tile_id image_x image_y n_votes spread version ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY

0 rows × 25 columns

source

get_blotches_for_tile


def get_blotches_for_tile(
    tile_id, version:NoneType=None
):

Call self as a function.

get_blotches_for_tile(tile_id, version="v3")
obsid marking_id angle tile_id image_x image_y n_votes radius_1 radius_2 vote_ratio ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY
323107 PSP_002942_0935 B04ee23 43.15 APF0001xbr 101.57 28868.46 7.0 29.92 16.73 0.69 ... 199.642 0.5 135.831931 -30.132724 215.557621 -3374.38611 -86.309439 -86.352639 97.95779 28

1 rows × 24 columns

get_blotches_for_tile(tile_id, version="v1")
obsid marking_id angle tile_id image_x image_y n_votes radius_1 radius_2 vote_ratio ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY

0 rows × 24 columns

source

get_hirise_id_for_tile


def get_hirise_id_for_tile(
    tile_id, version:NoneType=None
):

Call self as a function.

get_hirise_id_for_tile(tile_id)
# set_catalog_version changes the module default
assert _DEFAULT_VERSION == "v3.1"
set_catalog_version("v3")
assert _DEFAULT_VERSION == "v3"
set_catalog_version("v3.1")  # restore
assert _DEFAULT_VERSION == "v3.1"
# catalog_version context manager temporarily overrides and restores the default
assert _DEFAULT_VERSION == "v3.1"
with catalog_version("v1"):
    assert _DEFAULT_VERSION == "v1"
assert _DEFAULT_VERSION == "v3.1"  # restored after context exit
# catalog_version context manager restores even on exception
try:
    with catalog_version("v3"):
        assert _DEFAULT_VERSION == "v3"
        raise RuntimeError("simulated error")
except RuntimeError:
    pass
assert _DEFAULT_VERSION == "v3.1"  # restored despite exception
# _resolve_version returns explicit version when given, default otherwise, and normalises .0 suffixes
assert _resolve_version("v1") == "v1"
assert _resolve_version("v1.0") == "v1"
assert _resolve_version("v3.0") == "v3"
assert _resolve_version("v3.1") == "v3.1"
assert _resolve_version(None) == "v3.1"

Convenience joiners

Tiny helpers that left-join an obsid-keyed dataframe with metafull (MY, L_s) or get_region_names (roi_name).

source

attach_roi


def attach_roi(
    df, obsid_col:str='obsid', version:str='v3.1'
):

Left-join df with get_region_names(version=...)[obsid, roi_name].

source

attach_my


def attach_my(
    df, obsid_col:str='obsid', version:str='v3.1'
):

Left-join df with metafull[OBSERVATION_ID, MY, SOLAR_LONGITUDE].

The metafull columns are renamed to obsid, MY, l_s.