verify_output

In [1]:

    subcases_pattern="1_0486"
    source_name="Cyg X-1"
    osa_version='OSA11.0-dev210628.1813-17265'
    nscw=50
    ng_sig_limit=2.
    systematic_fraction=0.01
    emin_values="15,20,25,30,35"
    reference_instrument='spi'
    flux_tolerance=0.05

In [2]:

    # Parameters
    nscw = 50
    osa_version = "OSA11.0-dev210628.1813-17265"
    subcases_pattern = "Cyg_X-1_0684"
    systematic_fraction = 0.01

In [3]:

    !ls subcases/

    Cyg_X-1_0079  Cyg_X-1_0684  Cyg_X-1_1174	   Cyg_X-1_1554
    Cyg_X-1_0211  Cyg_X-1_0739  Cyg_X-1_1227_1_Nustar  Cyg_X-1_1626
    Cyg_X-1_0438  Cyg_X-1_0803  Cyg_X-1_1227_2_Nustar  Cyg_X-1_1684
    Cyg_X-1_0470  Cyg_X-1_0929  Cyg_X-1_1411	   Cyg_X-1_1684_Nustar
    Cyg_X-1_0486  Cyg_X-1_1052  Cyg_X-1_1416_Nustar    Cyg_X-1_1685_Nustar
    Cyg_X-1_0628  Cyg_X-1_1119  Cyg_X-1_1489	   Cyg_X-1_1742

# Cyg X-1 verification with SPI or NuSTAR reference¶

simultaneous observations allow easy comparisong

one should take care to have a compatible effective selection

In [4]:

    %matplotlib notebook
    emin_values_list=list(map(float, emin_values.split(",")))
    import matplotlib.pylab as plt

    import numpy as np
    import importlib
    #from astroquery.simbad import Simbad
    #from astropy import coordinates as coord
    import os
    import numpy as np
    import glob

    from IPython.display import Image
    from IPython.display import display
    import xspec
    import shutil

    import os
    import oda

    from scipy import stats
    import astropy.io.fits as fits

    WARNING:root:unable to setup logstash FileNotFoundError(2, 'No such file or directory')

In [5]:

    import odakb.sparql

    # pick only those that output OGIP
    r=odakb.sparql.select(query='''
        ?w oda:instrument oda:isgri;
           oda:referenceInstrument oda:%s
    '''%reference_instrument.lower())

    data_reduction_workflows = r

    if len(data_reduction_workflows) == 0:
        raise Exception('unable to find workflow for instruments isgri, %s'%reference_instrument)
    elif len(data_reduction_workflows) > 1:
        print('suspiciously many workflows for instruments isgri, %s'%reference_instrument)

    data_reduction_workflow = data_reduction_workflows[0]['w']

    data_reduction_workflow

    2021-07-21 20:55:00 7ac3e6378d3b oda.kb.sparql[224] INFO ODA Knowledge Base (SPARQL) root is https://www.astro.unige.ch/cdci/astrooda/dispatch-data/gw/odakb
    2021-07-21 20:55:00 7ac3e6378d3b oda.kb.sparql[224] INFO ODA KB responds 200

Out[5]:

    'http://odahub.io/cc/isgri-oda-spi-reference'

In [6]:

    data_reduction_workflows

Out[6]:

    [{'w': 'http://odahub.io/cc/isgri-oda-spi-reference'}]

In [7]:

    r=oda.evaluate("kb", data_reduction_workflow,
               subcases_pattern=subcases_pattern,
               reference_location=os.getcwd(),
               source_name=source_name,
               osa_version=osa_version,
               nscw=nscw)

    2021-07-21 20:55:00 7ac3e6378d3b odakb.sparql[224] INFO requested to resolve callable http://odahub.io/cc/isgri-oda-spi-reference
    2021-07-21 20:55:00 7ac3e6378d3b oda.kb.sparql[224] INFO ODA Knowledge Base (SPARQL) root is https://www.astro.unige.ch/cdci/astrooda/dispatch-data/gw/odakb
    2021-07-21 20:55:00 7ac3e6378d3b oda.kb.sparql[224] INFO ODA KB responds 200

    args ('http://odahub.io/cc/isgri-oda-spi-reference',)
    kwargs {'subcases_pattern': 'Cyg_X-1_0684', 'reference_location': '/tmp/nb2w-s6m5fh8d', 'source_name': 'Cyg X-1', 'osa_version': 'OSA11.0-dev210628.1813-17265', 'nscw': 50}

    2021-07-21 20:55:01 7ac3e6378d3b oda.kb.sparql[224] INFO ODA Knowledge Base (SPARQL) root is https://www.astro.unige.ch/cdci/astrooda/dispatch-data/gw/odakb
    2021-07-21 20:55:01 7ac3e6378d3b oda.kb.sparql[224] INFO ODA KB responds 200
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO found locations: [{'location': 'git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git'}, {'location': 'https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git'}, {'location': 'https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git'}]
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] WARNING no actual canonical query will default http://odahub.io/cc/isgri-oda-spi-reference
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO resolved callable:
     -- kind: http://odahub.io/callable/notebook
     -- origins: ['git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git']
     -- canonical_query: http://odahub.io/cc/isgri-oda-spi-reference
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] WARNING assuming this environment is compliant with http://odahub.io/cc/isgri-oda-spi-reference
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO fetch_origins '['git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git']' for query 'http://odahub.io/cc/isgri-oda-spi-reference' base dir query '/tmp/home-run/repo'
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO local copy: /tmp/nb2w-s6m5fh8d/code/odahub_io_cc_isgri-oda-spi-reference
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO origin git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git allowed by .*
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO trying to clone git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git as git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO failed to clone: Command '['git', 'clone', 'git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'code/odahub_io_cc_isgri-oda-spi-reference']' returned non-zero exit status 128.!
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO origin https://gitlab-ci-token:REDACTED@gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git allowed by .*
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO trying to clone git@gitlab.astro.unige.ch:integral/cc-workflows/cc-isgri-oda-spi-reference.git as https://gitlab-ci-token:REDACTED@gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO failed to clone: Command '['git', 'clone', 'https://gitlab-ci-token:REDACTED@gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'code/odahub_io_cc_isgri-oda-spi-reference']' returned non-zero exit status 128.!
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO local copy: /tmp/nb2w-s6m5fh8d/code/odahub_io_cc_isgri-oda-spi-reference
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO origin https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git allowed by .*
    2021-07-21 20:55:01 7ac3e6378d3b odakb.sparql[224] INFO trying to clone https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git as https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git
    2021-07-21 20:55:02 7ac3e6378d3b odakb.sparql[224] INFO failed to clone: Command '['git', 'clone', 'https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'code/odahub_io_cc_isgri-oda-spi-reference']' returned non-zero exit status 128.!
    2021-07-21 20:55:02 7ac3e6378d3b odakb.sparql[224] INFO origin https://gitlab-ci-token:REDACTED@gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git allowed by .*
    2021-07-21 20:55:02 7ac3e6378d3b odakb.sparql[224] INFO trying to clone https://gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git as https://gitlab-ci-token:REDACTED@gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git
    2021-07-21 20:55:02 7ac3e6378d3b odakb.sparql[224] INFO failed to clone: Command '['git', 'clone', 'https://gitlab-ci-token:REDACTED@gitlab.astro.unige.ch/integral/cc-workflows/cc-isgri-oda-spi-reference.git', 'code/odahub_io_cc_isgri-oda-spi-reference']' returned non-zero exit status 128.!
    2021-07-21 20:55:02 7ac3e6378d3b odakb.sparql[224] INFO local copy: /tmp/nb2w-s6m5fh8d/code/odahub_io_cc_isgri-oda-spi-reference
    2021-07-21 20:55:02 7ac3e6378d3b odakb.sparql[224] INFO origin https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git allowed by .*
    2021-07-21 20:55:02 7ac3e6378d3b odakb.sparql[224] INFO trying to clone https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git as https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git
    2021-07-21 20:55:03 7ac3e6378d3b odakb.sparql[224] INFO clonned succesfully!
    2021-07-21 20:55:03 7ac3e6378d3b odakb.sparql[224] INFO fetched origin for query http://odahub.io/cc/isgri-oda-spi-reference is https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git, all query names ['http://odahub.io/cc/isgri-oda-spi-reference', 'https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git', 'https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git']
    2021-07-21 20:55:03 7ac3e6378d3b odakb.sparql[224] INFO found oda.yaml: /tmp/nb2w-s6m5fh8d/oda.yaml
    2021-07-21 20:55:03 7ac3e6378d3b odakb.sparql[224] INFO found oda.yaml: /tmp/nb2w-s6m5fh8d/code/odahub_io_cc_isgri-oda-spi-reference/oda.yaml
    2021-07-21 20:55:03 7ac3e6378d3b odakb.sparql[224] INFO complete local context contains:
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO  --                          http://odahub.io/cc/cygx1: /tmp/nb2w-s6m5fh8d
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO  --                                 /tmp/home-run/repo: /tmp/nb2w-s6m5fh8d
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO  --        http://odahub.io/cc/isgri-oda-spi-reference: /tmp/nb2w-s6m5fh8d/code/odahub_io_cc_isgri-oda-spi-reference
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO  -- https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git: /tmp/nb2w-s6m5fh8d/code/odahub_io_cc_isgri-oda-spi-reference
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO bucket name odahub-io-cc-isgri-oda-spi-reference-default-145c6ca-dd4c46c2
    2021-07-21 20:55:04 7ac3e6378d3b odakb.datalake[224] INFO discovered minio URL with environment variable (MINIO_URL):minio.obsuks1.unige.ch
    2021-07-21 20:55:04 7ac3e6378d3b odakb.datalake[224] INFO discovered minio access key with environment variable (MINIO_KEY)
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO will execute (evaluate with local context)
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO                 path: /tmp/nb2w-s6m5fh8d/code/odahub_io_cc_isgri-oda-spi-reference
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO              version: 145c6ca
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO               origin: https://github.com/integral-observatory/cc-isgri-oda-spi-reference.git
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO        callable_kind: http://odahub.io/callable/notebook
    2021-07-21 20:55:04 7ac3e6378d3b odakb.sparql[224] INFO                  oda: {'uri_base': 'http://odahub.io/cc/isgri-oda-spi-reference', 'metadata': {'source_short_name': 'undefined'}}
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg Cyg_X-1_0684
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg Cyg_X-1_0684 provided as Cyg_X-1_0684
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg /tmp/nb2w-s6m5fh8d
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg /tmp/nb2w-s6m5fh8d provided as /tmp/nb2w-s6m5fh8d
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg Cyg X-1
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg Cyg X-1 provided as Cyg X-1
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg OSA11.0-dev210628.1813-17265
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg OSA11.0-dev210628.1813-17265 provided as OSA11.0-dev210628.1813-17265
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg 50
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO request arg 50 provided as 50
    2021-07-21 20:55:04 7ac3e6378d3b root[224] INFO found parameters {'subcases_pattern': 'Cyg_X-1_0684', 'reference_location': '/tmp/nb2w-s6m5fh8d', 'source_name': 'Cyg X-1', 'osa_version': 'OSA11.0-dev210628.1813-17265', 'nscw': 50}
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO starting job
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO new tmpdir: /tmp/nb2w-3okw827f
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO git clone output: b"Cloning into '/tmp/nb2w-3okw827f'...\ndone.\n"
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO provided notebook nbformat version minor 2 while nbformat package minor version 5
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] WARNING will attempt to convert, but expect other warnings!
    2021-07-21 20:55:04 7ac3e6378d3b nb2workflow.nbadapter[224] INFO stored pre-processed notebook as /tmp/nb2w-3okw827f/isgri-oda-spi-reference_preproc.ipynb
    2021-07-21 20:55:04 7ac3e6378d3b papermill[224] INFO Input Notebook:  /tmp/nb2w-3okw827f/isgri-oda-spi-reference_preproc.ipynb
    2021-07-21 20:55:04 7ac3e6378d3b papermill[224] INFO Output Notebook: /tmp/nb2w-3okw827f/isgri-oda-spi-reference_output.ipynb
    2021-07-21 20:55:04 7ac3e6378d3b papermill[224] INFO Working directory: /tmp/nb2w-3okw827f
    2021-07-21 20:55:04 7ac3e6378d3b blib2to3.pgen2.driver[224] INFO Generating grammar tables from /pyenv/versions/3.8.2/lib/python3.8/site-packages/blib2to3/Grammar.txt
    2021-07-21 20:55:04 7ac3e6378d3b blib2to3.pgen2.driver[224] INFO Writing grammar tables to /tmp/home-run/.cache/black/21.7b0/Grammar3.8.2.final.0.pickle
    2021-07-21 20:55:04 7ac3e6378d3b blib2to3.pgen2.driver[224] INFO Writing failed: [Errno 2] No such file or directory: '/tmp/home-run/.cache/black/21.7b0/tmpnlz8xuiv'
    2021-07-21 20:55:04 7ac3e6378d3b blib2to3.pgen2.driver[224] INFO Generating grammar tables from /pyenv/versions/3.8.2/lib/python3.8/site-packages/blib2to3/PatternGrammar.txt
    2021-07-21 20:55:04 7ac3e6378d3b blib2to3.pgen2.driver[224] INFO Writing grammar tables to /tmp/home-run/.cache/black/21.7b0/PatternGrammar3.8.2.final.0.pickle
    2021-07-21 20:55:04 7ac3e6378d3b blib2to3.pgen2.driver[224] INFO Writing failed: [Errno 2] No such file or directory: '/tmp/home-run/.cache/black/21.7b0/tmpr6xzdmy5'
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Executing notebook with kernel: python3
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Executing Cell 1---------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Ending Cell 1------------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Executing Cell 2---------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Ending Cell 2------------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Executing Cell 3---------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Ending Cell 3------------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Executing Cell 4---------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Ending Cell 4------------------------------------------
    2021-07-21 20:55:05 7ac3e6378d3b papermill[224] INFO Executing Cell 5---------------------------------------
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO Ending Cell 5------------------------------------------
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO Executing Cell 6---------------------------------------
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO found subcase in reference location dir: /tmp/nb2w-s6m5fh8d/subcases/Cyg_X-1_0684
    read reference sources:          X        Y          RA        Dec  peak_counts       RMS  exposure  \
    0  385.000  392.000  299.875974  40.764416     0.770074  0.169023   60171.7
    1  307.907  353.738  308.060532  37.640237     8.284580  0.153424   60144.9
    2  390.959  324.362  299.587968  35.198259   169.280000  0.178778   60181.6
    3  309.115  394.211  308.107221  40.960882    12.468900  0.173089   60116.8

       arcmin_to_cat   association
    0       1.867553         Cyg A
    1       0.210749  EXO 2030+375
    2       0.231689       Cyg X-1
    3       0.188352       Cyg X-3
    found /tmp/nb2w-s6m5fh8d/subcases/Cyg_X-1_0684 spectrum__photo_Cyg_X-1_avrg.fits 49 4          X        Y          RA        Dec  peak_counts       RMS  exposure  \
    0  385.000  392.000  299.875974  40.764416     0.770074  0.169023   60171.7
    1  307.907  353.738  308.060532  37.640237     8.284580  0.153424   60144.9
    2  390.959  324.362  299.587968  35.198259   169.280000  0.178778   60181.6
    3  309.115  394.211  308.107221  40.960882    12.468900  0.173089   60116.8

       arcmin_to_cat   association
    0       1.867553         Cyg A
    1       0.210749  EXO 2030+375
    2       0.231689       Cyg X-1
    3       0.188352       Cyg X-3

    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO ('spectrum__photo_Cyg_X-1_avrg.fits',
     'rmf__photo_Cyg_X-1_avrg.fits',
     ['068400130010',
      '068400140010',
      '068400150010',
      '068400160010',
      '068400170010',
      '068400180010',
      '068400190010',
      '068400200010',
      '068400210010',
      '068400320010',
      '068400330010',
      '068400340010',
      '068400350010',
      '068400360010',
      '068400370010',
      '068400380010',
      '068400390010',
      '068400400010',
      '068400410010',
      '068400420010',
      '068400520010',
      '068400530010',
      '068400540010',
      '068400550010',
      '068400560010',
      '068400570010',
      '068400580010',
      '068400590010',
      '068400600010',
      '068400610010',
      '068400720010',
      '068400730010',
      '068400740010',
      '068400750010',
      '068400760010',
      '068400770010',
      '068400780010',
      '068400790010',
      '068400800010',
      '068400810010',
      '068400920010',
      '068400930010',
      '068400940010',
      '068400950010',
      '068400960010',
      '068400970010',
      '068400980010',
      '068400990010',
      '068401000010'])
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO Ending Cell 6------------------------------------------
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO Executing Cell 7---------------------------------------
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO          X        Y          RA        Dec  peak_counts       RMS  exposure  \
    0  385.000  392.000  299.875974  40.764416     0.770074  0.169023   60171.7
    1  307.907  353.738  308.060532  37.640237     8.284580  0.153424   60144.9
    2  390.959  324.362  299.587968  35.198259   169.280000  0.178778   60181.6
    3  309.115  394.211  308.107221  40.960882    12.468900  0.173089   60116.8

       arcmin_to_cat   association
    0       1.867553         Cyg A
    1       0.210749  EXO 2030+375
    2       0.231689       Cyg X-1
    3       0.188352       Cyg X-3
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO Ending Cell 7------------------------------------------
    2021-07-21 20:55:06 7ac3e6378d3b papermill[224] INFO Executing Cell 8---------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO <Table length=1>
     MAIN_ID        RA           DEC      ... COO_WAVELENGTH     COO_BIBCODE
                 "h:m:s"       "d:m:s"    ...
      object      str13         str13     ...      str1             object
    --------- ------------- ------------- ... -------------- -------------------
    HD 226868 19 58 21.6758 +35 12 05.782 ...              O 2018yCat.1345....0G
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Ending Cell 8------------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Executing Cell 9---------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO nscw (50) > than available scw (49), using them all
    832

    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO '068400130010.001,068400140010.001,068400150010.001,068400160010.001,068400170010.001,068400180010.001,068400190010.001,068400200010.001,068400210010.001,068400320010.001,068400330010.001,068400340010.001,068400350010.001,068400360010.001,068400370010.001,068400380010.001,068400390010.001,068400400010.001,068400410010.001,068400420010.001,068400520010.001,068400530010.001,068400540010.001,068400550010.001,068400560010.001,068400570010.001,068400580010.001,068400590010.001,068400600010.001,068400610010.001,068400720010.001,068400730010.001,068400740010.001,068400750010.001,068400760010.001,068400770010.001,068400780010.001,068400790010.001,068400800010.001,068400810010.001,068400920010.001,068400930010.001,068400940010.001,068400950010.001,068400960010.001,068400970010.001,068400980010.001,068400990010.001,068401000010.001'
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Ending Cell 9------------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Executing Cell 10--------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO will find appropriate API for OSA version OSA11.0-dev210628.1813-17265
    osa version has subversion - will use staging-1-3

    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Ending Cell 10-----------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Executing Cell 11--------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Ending Cell 11-----------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO Executing Cell 12--------------------------------------
    2021-07-21 20:55:07 7ac3e6378d3b papermill[224] INFO We make a mosaic to extract a catalog

    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Ending Cell 12-----------------------------------------
    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Executing Cell 13--------------------------------------
    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Ending Cell 13-----------------------------------------
    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Executing Cell 14--------------------------------------
    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO {"cat_frame": "fk5", "cat_coord_units": "deg", "cat_column_list": [[14, 16, 17], ["Cyg X-1", "Cyg X-3", "EXO 2030+375"], [1113.8607177734375, 74.38999938964844, 56.354148864746094], [299.5887756347656, 308.109130859375, 308.0623779296875], [35.19800567626953, 40.9603157043457, 37.64250564575195], [-32768, -32768, -32768], [2, 2, 2], [0, 0, 0], [0.0002800000074785203, 0.0002800000074785203, 0.0005600000149570405]], "cat_column_names": ["meta_ID", "src_names", "significance", "ra", "dec", "NEW_SOURCE", "ISGRI_FLAG", "FLAG", "ERR_RAD"], "cat_column_descr": [["meta_ID", "<i8"], ["src_names", "<U20"], ["significance", "<f8"], ["ra", "<f8"], ["dec", "<f8"], ["NEW_SOURCE", "<i8"], ["ISGRI_FLAG", "<i8"], ["FLAG", "<i8"], ["ERR_RAD", "|O"]], "cat_lat_name": "dec", "cat_lon_name": "ra"}

    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Ending Cell 14-----------------------------------------
    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Executing Cell 15--------------------------------------
    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Ending Cell 15-----------------------------------------
    2021-07-21 21:11:52 7ac3e6378d3b papermill[224] INFO Executing Cell 16--------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Ending Cell 16-----------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Executing Cell 17--------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO {'src_name': 'Cyg X-1', 'product': 'isgri_spectrum'}
    {'src_name': 'Cyg X-1', 'product': 'isgri_arf'}
    {'src_name': 'Cyg X-1', 'product': 'isgri_rmf'}
    {'src_name': 'Cyg X-3', 'product': 'isgri_spectrum'}
    {'src_name': 'Cyg X-3', 'product': 'isgri_arf'}
    {'src_name': 'Cyg X-3', 'product': 'isgri_rmf'}
    {'src_name': 'EXO 2030+375', 'product': 'isgri_spectrum'}
    {'src_name': 'EXO 2030+375', 'product': 'isgri_arf'}
    {'src_name': 'EXO 2030+375', 'product': 'isgri_rmf'}
    {'src_name': 'Background', 'product': 'isgri_spectrum'}
    {'src_name': 'Background', 'product': 'isgri_arf'}
    {'src_name': 'Background', 'product': 'isgri_rmf'}

    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Ending Cell 17-----------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Executing Cell 18--------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Ending Cell 18-----------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Executing Cell 19--------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Ending Cell 19-----------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Executing Cell 20--------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Ending Cell 20-----------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Executing Cell 21--------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Ending Cell 21-----------------------------------------
    2021-07-21 21:31:17 7ac3e6378d3b papermill[224] INFO Executing Cell 22--------------------------------------
    2021-07-21 21:31:18 7ac3e6378d3b papermill[224] WARNING WARNING:root:unable to setup logstash FileNotFoundError(2, 'No such file or directory')

    2021-07-21 21:31:18 7ac3e6378d3b papermill[224] INFO Ending Cell 22-----------------------------------------
    2021-07-21 21:31:18 7ac3e6378d3b root[224] INFO execution SUCCESSFUL!
    2021-07-21 21:31:20 7ac3e6378d3b root[224] INFO converting... 0
    2021-07-21 21:31:20 7ac3e6378d3b nb2workflow.nbadapter[224] INFO exported html to isgri-oda-spi-reference_output.html
    2021-07-21 21:31:33 7ac3e6378d3b odakb.datalake[224] INFO discovered minio URL with environment variable (MINIO_URL):minio.obsuks1.unige.ch
    2021-07-21 21:31:33 7ac3e6378d3b odakb.datalake[224] INFO discovered minio access key with environment variable (MINIO_KEY)
    2021-07-21 21:31:33 7ac3e6378d3b odakb.datalake[224] ERROR error removing bucket odahub-io-cc-isgri-oda-spi-reference-default-145c6ca-dd4c46c2: NoSuchBucket: message: The specified bucket does not exist.
    2021-07-21 21:31:37 7ac3e6378d3b odakb.sparql[224] INFO successfully stored to the datalake odahub-io-cc-isgri-oda-spi-reference-default-145c6ca-dd4c46c2
    2021-07-21 21:31:37 7ac3e6378d3b root[224] INFO ({'event': 'done'},); {}

    writing reference_spec.fits
    writing reference_rmf.fits
    writing /tmp/isgri_spectrum_Cyg_X-1.fits
    writing /tmp/isgri_arf_Cyg_X-1.fits
    writing /tmp/isgri_rmf_Cyg_X-1.fits
    writing isgri-oda-spi-reference_output.html
    trying to decode json reference_spec
    trying to decode json reference_spec_content
    trying to decode json reference_rmf
    trying to decode json reference_rmf_content
    trying to decode json isgri_spec
    trying to decode json isgri_spec_content
    trying to decode json isgri_arf
    trying to decode json isgri_arf_content
    trying to decode json isgri_rmf
    trying to decode json isgri_rmf_content
    trying to decode json output_notebook_html
    trying to decode json output_notebook_html_content

In [8]:

    for k,v in r.items():
        if not k.endswith("_content"):
            print(k,v)

    reference_spec reference_spec.fits
    reference_rmf reference_rmf.fits
    isgri_spec /tmp/isgri_spectrum_Cyg_X-1.fits
    isgri_arf /tmp/isgri_arf_Cyg_X-1.fits
    isgri_rmf /tmp/isgri_rmf_Cyg_X-1.fits
    output_notebook_html isgri-oda-spi-reference_output.html

In [9]:

    import ccspectools
    importlib.reload(ccspectools)

    spec_summary = ccspectools.view_group_spectra(r, reference_instrument, subcases_pattern,
                                                  systematic_fraction=systematic_fraction)

    reference exposure 91561.9686142041 isgri exposure 59624.60431213378 isgri time range 54606.5915--54608.9359 N/A

In [10]:

    ebounds = fits.open(r['isgri_rmf'])['EBOUNDS'].data
    d_e = ebounds['E_MAX'] - ebounds['E_MIN']
    c_e = (ebounds['E_MAX'] * ebounds['E_MIN'])**0.5

In [11]:

    def extra_ticks(ax=None, extra_ticks=None):
        if ax is None:
            ax = plt.gca()

        if extra_ticks is None:
            extra_ticks = [15, 20]

        all_ticks = extra_ticks + list(ax.get_xticks())

        ax.set_xticks(all_ticks)
        ax.set_xticklabels(map(str, all_ticks))

In [12]:

    from astropy.io import fits

    def patch_problematic_spectrum_values(fn):
        f = fits.open(fn)
        d = f[1].data

        m_bad = np.isnan(d['RATE']) | np.isinf(d['STAT_ERR'])

        d['QUALITY'] = 0
        d['RATE'][m_bad] = 0
        d['STAT_ERR'][m_bad] = 0
        d['QUALITY'][m_bad] = 3

        specname_patched = fn.replace(".fits", "_patched.fits")
        f.writeto(specname_patched, overwrite=True)
        return specname_patched

    f = fits.open(patch_problematic_spectrum_values(r['isgri_spec']))[1].data

    plt.figure()

    m = f['QUALITY'] == 0

    plt.plot(c_e[m], (f['RATE']/d_e)[m])

    # check value
    # normalize efficiency here

    plt.loglog()

    extra_ticks(None, [15, 20, 25, 30])
    plt.xlim([14, 600])

    plt.grid()

In [13]:

    import importlib
    importlib.reload(ccspectools)

    def model_setter(r, reference_instrument, c_emin):
        if reference_instrument == 'spi':
            model='cflux*(cutoffpl)'
        elif reference_instrument.lower() == 'nustar':
            model='cflux*(cutoffpl)'

        m1=xspec.Model(model)

        specname_patched = patch_problematic_spectrum_values(r['isgri_spec'])

        if reference_instrument == 'spi':
            xspec.AllData(specname_patched+" 2:2 " + r["reference_spec"] )
        elif reference_instrument.lower() == 'nustar':
            xspec.AllData(specname_patched+" 2:2 " + r["reference_spec_A"] + " 3:3 " + r["reference_spec_B"] )
        else:
            raise ValueError

        s     = xspec.AllData(1)
        s_ref = xspec.AllData(2)

        isgri = xspec.AllModels(1)
        ref   = xspec.AllModels(2)

        print(m1.nParameters)

        xspec.AllData.ignore('bad')
        xspec.AllData.ignore('600.0-**')

        ig="**-%.2f,600.-**"%c_emin
        print(ig)
        s.ignore(ig)

        if reference_instrument.lower() == 'spi':
            ig="**-23.0,600.0-**"
            print(reference_instrument, ig)
            s_ref.ignore(ig)

        if reference_instrument.lower() == 'nustar':
            ig="**-15.0,70.0-**"
            print(reference_instrument, ig)
            s_ref.ignore(ig)

            s_ref_b=xspec.AllData(3)
            ref_b=xspec.AllModels(3)

            s_ref_b.ignore(ig)

        ref.cflux.lg10Flux.untie()
        ref.cflux.lg10Flux.frozen=False

        isgri.cflux.lg10Flux=-8
        ref.cflux.lg10Flux=-8

        if reference_instrument.lower() == 'nustar':
            ref_b.cflux.lg10Flux.untie()
            ref_b.cflux.lg10Flux.frozen=False
            #ref_b.gabs.LineE.link="%d"%(m1.nParameters+ref_b.gabs.LineE.index)
            ref_b.cutoffpl.HighECut.link="%d"%(m1.nParameters+ref_b.cutoffpl.HighECut.index)
            ref_b.cflux.lg10Flux=-8

        isgri.cflux.Emin=40.
        isgri.cflux.Emax=100.
        isgri.cutoffpl.norm=1.0
        isgri.cutoffpl.norm.frozen=True

    #     isgri.powerlaw.PhoIndex=2
    #     isgri.powerlaw.PhoIndex.frozen=True

        isgri.cutoffpl.HighECut=50.0
        isgri.cutoffpl.HighECut.frozen=False

        xspec.Fit.perform()

    #     if reference_instrument.lower() == 'nustar':
    #         isgri.highecut.foldE.frozen=False
    #         isgri.highecut.cutoffE.frozen=False

        ref.cutoffpl.HighECut.untie()

        xspec.Fit.perform()

        if reference_instrument == 'Nustar':
            return isgri, [ref,ref_b]
        else:
            return isgri, [ref]

    fit_by_lt, fn_by_lt = ccspectools.fit(
        data=r,
        reference_instrument = reference_instrument,
        model_setter = model_setter,
        emin_values = emin_values_list,
        fn_prefix=subcases_pattern
    )

    6
    **-15.00,600.-**
    spi **-23.0,600.0-**
    lg10Flux -7.74 False False
    PhoIndex 1.23 False False
    HighECut 114.47 False False
    lg10Flux -7.91 False False
    HighECut 110.42 False False

    6
    **-20.00,600.-**
    spi **-23.0,600.0-**
    lg10Flux -7.74 False False
    PhoIndex 1.25 False False
    HighECut 116.64 False False
    lg10Flux -7.91 False False
    HighECut 112.69 False False

    6
    **-25.00,600.-**
    spi **-23.0,600.0-**
    lg10Flux -7.74 False False
    PhoIndex 1.34 False False
    HighECut 134.16 False False
    lg10Flux -7.92 False False
    HighECut 133.63 False False

    6
    **-30.00,600.-**
    spi **-23.0,600.0-**
    lg10Flux -7.74 False False
    PhoIndex 1.36 False False
    HighECut 137.01 False False
    lg10Flux -7.92 False False
    HighECut 137.36 False False

    6
    **-35.00,600.-**
    spi **-23.0,600.0-**
    lg10Flux -7.74 False False
    PhoIndex 1.35 False False
    HighECut 135.24 False False
    lg10Flux -7.92 False False
    HighECut 134.35 False False

In [14]:

    r.keys()

Out[14]:

    dict_keys(['reference_spec', 'reference_spec_content', 'reference_rmf', 'reference_rmf_content', 'isgri_spec', 'isgri_spec_content', 'isgri_arf', 'isgri_arf_content', 'isgri_rmf', 'isgri_rmf_content', 'output_notebook_html', 'output_notebook_html_content'])

In [15]:

    fit_by_lt

Out[15]:

    {'15': {'emin': 15.0,
      'chi2_red': 3.3198404100764525,
      'chi2': 743.6442518571254,
      'ndof': 224,
      'models': {'isgri': {'lg10Flux_03': [-7.735392490859771,
         -7.7362353144406715,
         -7.734552470973699],
        'PhoIndex_04': [1.2331828809072005, 1.223758587219589, 1.2425652752573972],
        'HighECut_05': [114.46815797350604, 112.388226831115, 116.61812015425214]},
       'ref': {'lg10Flux_03': [-7.911238687993719,
         -7.91236066769297,
         -7.910120444649718],
        'HighECut_05': [110.41546320637379,
         108.57338457033498,
         112.31037125475211]}}},
     '20': {'emin': 20.0,
      'chi2_red': 2.936803183579311,
      'chi2': 649.0335035710277,
      'ndof': 221,
      'models': {'isgri': {'lg10Flux_03': [-7.735697498968773,
         -7.736542453094561,
         -7.734855289139392],
        'PhoIndex_04': [1.2459977675685512,
         1.2362527701308041,
         1.2556962889211176],
        'HighECut_05': [116.64291266965583,
         114.45057873921222,
         118.91129003078485]},
       'ref': {'lg10Flux_03': [-7.911879079145289,
         -7.913009103328287,
         -7.910752706591198],
        'HighECut_05': [112.69371685987605,
         110.73345358372214,
         114.712743941991]}}},
     '25': {'emin': 25.0,
      'chi2_red': 1.3529125113952811,
      'chi2': 292.2291024613807,
      'ndof': 216,
      'models': {'isgri': {'lg10Flux_03': [-7.7364321896803245,
         -7.73727347231434,
         -7.735593197370045],
        'PhoIndex_04': [1.3449844010641008,
         1.3342208084426264,
         1.3556964828200393],
        'HighECut_05': [134.15518725120796,
         131.18762725102195,
         137.24081888974695]},
       'ref': {'lg10Flux_03': [-7.917089757944747,
         -7.918260441270992,
         -7.915923296442188],
        'HighECut_05': [133.63132814469847,
         130.7527281909707,
         136.61406404683777]}}},
     '30': {'emin': 30.0,
      'chi2_red': 1.2440848496010104,
      'chi2': 262.5019032658132,
      'ndof': 211,
      'models': {'isgri': {'lg10Flux_03': [-7.736429776845952,
         -7.7372724981631515,
         -7.735589495879257],
        'PhoIndex_04': [1.3597255322906365, 1.347669656393452, 1.3717139833498886],
        'HighECut_05': [137.0054654264238,
         133.77689026840272,
         140.37004701623835]},
       'ref': {'lg10Flux_03': [-7.917907614576544,
         -7.919119738475933,
         -7.916699852125358],
        'HighECut_05': [137.36205040089717,
         134.04486308813762,
         140.81393391843883]}}},
     '35': {'emin': 35.0,
      'chi2_red': 1.2483772570622707,
      'chi2': 257.16571495482776,
      'ndof': 206,
      'models': {'isgri': {'lg10Flux_03': [-7.736855380486416,
         -7.737723103298391,
         -7.735990692321955],
        'PhoIndex_04': [1.3478716693117068,
         1.3344293218928787,
         1.3611892041392044],
        'HighECut_05': [135.2413073182734,
         131.97275871673125,
         138.64920608587326]},
       'ref': {'lg10Flux_03': [-7.917249078338554,
         -7.918500349953762,
         -7.916002455450509],
        'HighECut_05': [134.34741153055043,
         130.86128375470182,
         137.98404258820207]}}}}

In [16]:

    def basic_consistency(fit_by_lt, nh_sig_limit):
        # marginally more adcanced choice of low thresholds
        print("lt\tchi2_r\tprob\tsigmas")
        for lt,d in fit_by_lt.items():
            d['nh_prob']=stats.chi2(d['ndof']).sf(d['chi2'])
            d['nh_sig']=stats.norm().isf(d['nh_prob'])
            print("%.1f\t%.2f\t%.2f\t%.2f"%(float(lt), d['chi2_red'], d['nh_prob'], d['nh_sig']) )

        #good_lt = min([p for p in fit_by_lt.items() if p[1]['nh_sig']< nh_sig_limit], key=lambda x:x)
        good_lt = min([p for p in fit_by_lt.items() ], key=lambda x:x[1]['chi2_red'])

        return good_lt

    good_lt = basic_consistency(fit_by_lt, ng_sig_limit)

    lt	chi2_r	prob	sigmas
    15.0	3.32	0.00	15.87
    20.0	2.94	0.00	13.81
    25.0	1.35	0.00	3.34
    30.0	1.24	0.01	2.36
    35.0	1.25	0.01	2.37

In [17]:

    good_lt

Out[17]:

    ('30',
     {'emin': 30.0,
      'chi2_red': 1.2440848496010104,
      'chi2': 262.5019032658132,
      'ndof': 211,
      'models': {'isgri': {'lg10Flux_03': [-7.736429776845952,
         -7.7372724981631515,
         -7.735589495879257],
        'PhoIndex_04': [1.3597255322906365, 1.347669656393452, 1.3717139833498886],
        'HighECut_05': [137.0054654264238,
         133.77689026840272,
         140.37004701623835]},
       'ref': {'lg10Flux_03': [-7.917907614576544,
         -7.919119738475933,
         -7.916699852125358],
        'HighECut_05': [137.36205040089717,
         134.04486308813762,
         140.81393391843883]}},
      'nh_prob': 0.00914191360042112,
      'nh_sig': 2.3598195333301364})

In [18]:

    #Status for Low thresholds
    status='OK'
    try:
        print(good_lt[0])
    except:
        status='NOK'
    print("Goodness of fit test result")

    30
    Goodness of fit test result

In [19]:

    importlib.reload(ccspectools)
    parameter_comparison = ccspectools.parameter_comparison(good_lt, ng_sig_limit,
                                                            reference_instrument=reference_instrument,
                                                           flux_tolerance=flux_tolerance)

    lg10Flux_03 -7.74 +/- 0.00 ; -7.92 +/- 0.00 ; 123.2
    Using a fractional difference of 0.518720 to compare lg10Flux_03 for spi
    HighECut_05 137.01 +/- 3.36 ; 137.36 +/- 3.32 ; -0.1

In [20]:

    parameter_comparison

Out[20]:

    {'lg10Flux_03': {'ref': [-7.917907614576544, 0.001207762451185701],
      'isgri': [-7.736429776845952, 0.0008427213171993131],
      'significance': 123.22729058145549,
      'success': False},
     'HighECut_05': {'ref': [137.36205040089717, 3.3171873127595575],
      'isgri': [137.0054654264238, 3.3645815898145486],
      'significance': 0.07547022788379086,
      'success': True}}

In [21]:

    reference_instrument, good_lt

Out[21]:

    ('spi',
     ('30',
      {'emin': 30.0,
       'chi2_red': 1.2440848496010104,
       'chi2': 262.5019032658132,
       'ndof': 211,
       'models': {'isgri': {'lg10Flux_03': [-7.736429776845952,
          -7.7372724981631515,
          -7.735589495879257],
         'PhoIndex_04': [1.3597255322906365,
          1.347669656393452,
          1.3717139833498886],
         'HighECut_05': [137.0054654264238,
          133.77689026840272,
          140.37004701623835]},
        'ref': {'lg10Flux_03': [-7.917907614576544,
          -7.919119738475933,
          -7.916699852125358],
         'HighECut_05': [137.36205040089717,
          134.04486308813762,
          140.81393391843883]}},
       'nh_prob': 0.00914191360042112,
       'nh_sig': 2.3598195333301364}))

In [22]:

    #Status for parameters
    status='OK'
    for par, par_res in parameter_comparison.items():
        if not par_res['success']:
            status='NOK'
    if good_lt[1]['nh_sig'] > ng_sig_limit:
        status='NOK'
    print("Parameter test result " + status)

    Parameter test result NOK

In [23]:

    fn_picked=fn_by_lt[good_lt[0]]
    fn_picked

Out[23]:

    'fit_lt30.png'

In [24]:

    summary={
        'status': status,
        'isgri_exposure': spec_summary['isgri_exposure'],
        'reference_exposure': spec_summary['reference_exposure'],
        'isgri_times': spec_summary['isgri_times'],
        'reference_times':spec_summary['reference_times'],
        'parameter_comparison': parameter_comparison,
    }
    summary

Out[24]:

    {'status': 'NOK',
     'isgri_exposure': 59624.60431213378,
     'reference_exposure': 91561.9686142041,
     'isgri_times': '54606.5915--54608.9359',
     'reference_times': 'N/A',
     'parameter_comparison': {'lg10Flux_03': {'ref': [-7.917907614576544,
        0.001207762451185701],
       'isgri': [-7.736429776845952, 0.0008427213171993131],
       'significance': 123.22729058145549,
       'success': False},
      'HighECut_05': {'ref': [137.36205040089717, 3.3171873127595575],
       'isgri': [137.0054654264238, 3.3645815898145486],
       'significance': 0.07547022788379086,
       'success': True}}}

In [25]:

    summary=summary
    fit_results=fit_by_lt
    good_fit_png=fn_picked

In [26]:

    import papermill as pm
    import scrapbook as sb
    import base64
    import json
    import os

    from nb2workflow.nbadapter import denumpyfy

    try:
        sb.glue("summary",denumpyfy(summary))
    except Exception as e:
        print("failed to glue summary", summary)
        print("will glue jsonified")
        sb.glue("summary",json.dumps(denumpyfy(summary)))

    isinstance(summary,str) and os.path.exists(summary) and sb.glue("summary_content",base64.b64encode(open(summary,'rb').read()).decode())

    try:
        sb.glue("fit_results",denumpyfy(fit_results))
    except Exception as e:
        print("failed to glue fit_results", fit_results)
        print("will glue jsonified")
        sb.glue("fit_results",json.dumps(denumpyfy(fit_results)))

    isinstance(fit_results,str) and os.path.exists(fit_results) and sb.glue("fit_results_content",base64.b64encode(open(fit_results,'rb').read()).decode())

    try:
        sb.glue("good_fit_png",denumpyfy(good_fit_png))
    except Exception as e:
        print("failed to glue good_fit_png", good_fit_png)
        print("will glue jsonified")
        sb.glue("good_fit_png",json.dumps(denumpyfy(good_fit_png)))

    isinstance(good_fit_png,str) and os.path.exists(good_fit_png) and sb.glue("good_fit_png_content",base64.b64encode(open(good_fit_png,'rb').read()).decode())