Add retrieval function for BSRN data (#1254)

* Initial commit

* Fix typo in doc string

* Fix stickler

* Update api.rst, __init__.py, and whatsnew

* Major refactoring

* Coverage for test_bsrn

* Coverage for Warnings in case of no files avaiable

* Fix stickler

* Correct test_get_bsrn_bad_station test

* Specify warning category

* Update dates used in test_get_bsrn_no_files

* Add secret credentials for testing

* Documentation updates

* Move line_no_dict 7 lines down

* Add requires_bsrn_credentials to conftest.py

* Add parsing of logical records 0300 and 0500

* Raise os.environ as ValueError for debugging

* Export BSRN credentials in conda_linux.yml

* Add parse_bsrn

* Fix stickler and minor doc changes

* Coverage for additional logical records

* Refactor warnings in get_bsrn

If only some files are missing, give only one warning with a list of missing files

* Add Hint section

* Add function for empty dataframe and restructure data docs

* Add gri to list of variables

* Coverage for records not found

* Coverage for no logical records found

* Formatting of data columns table

* Merge read_ and get_bsrn in whatsnew

* Changes from review by kandersol-nrel

* Add lat/lon ISO 19115 convention to metadata

* Add bio.seek(0) to get_bsrn()

Author: AdamRJensen

ci/azure/conda_linux.yml

@@ -38,6 +38,8 @@ jobs:
   - script: |
       source activate test_env
       export NREL_API_KEY=$(nrelApiKey)
+      export BSRN_FTP_USERNAME=$(BSRN_FTP_USERNAME)
+      export BSRN_FTP_PASSWORD=$(BSRN_FTP_PASSWORD)
       pytest pvlib --remote-data --junitxml=junit/test-results.xml --cov --cov-report=xml --cov-report=html
     displayName: 'pytest'
   - task: PublishTestResults@2

docs/sphinx/source/api.rst

@@ -491,7 +491,9 @@ of sources and file formats relevant to solar energy modeling.
    iotools.read_pvgis_tmy
    iotools.get_pvgis_hourly
    iotools.read_pvgis_hourly
+   iotools.get_bsrn
    iotools.read_bsrn
+   iotools.parse_bsrn
    iotools.get_cams
    iotools.read_cams
    iotools.parse_cams

docs/sphinx/source/whatsnew/v0.9.0.rst

@@ -111,8 +111,9 @@ Enhancements
   :func:`~pvlib.iotools.get_pvgis_hourly` for reading and retrieving hourly
   solar radiation data and PV power output from PVGIS. (:pull:`1186`,
   :issue:`849`)
-* Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data
-  files. (:pull:`1145`, :issue:`1015`)
+* Add :func:`~pvlib.iotools.get_bsrn` and :func:`~pvlib.iotools.read_bsrn`
+  for retrieving and reading BSRN solar radiation data files.
+  (:pull:`1254`, :pull:`1145`, :issue:`1015`)
 * Add :func:`~pvlib.iotools.get_cams`,
   :func:`~pvlib.iotools.parse_cams`, and 
   :func:`~pvlib.iotools.read_cams`

pvlib/data/variables_style_rules.csv

@@ -7,6 +7,7 @@ dni_extra;direct normal irradiance at top of atmosphere (extraterrestrial)
 dhi;diffuse horizontal irradiance
 bhi;beam/direct horizontal irradiance
 ghi;global horizontal irradiance
+gri;ground-reflected irradiance
 aoi;angle of incidence between :math:`90\deg` and :math:`90\deg`
 aoi_projection;cos(aoi)
 airmass;airmass

pvlib/iotools/__init__.py

@@ -15,7 +15,9 @@
 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy  # noqa: F401
 from pvlib.iotools.pvgis import read_pvgis_hourly  # noqa: F401
 from pvlib.iotools.pvgis import get_pvgis_hourly  # noqa: F401
+from pvlib.iotools.bsrn import get_bsrn  # noqa: F401
 from pvlib.iotools.bsrn import read_bsrn  # noqa: F401
+from pvlib.iotools.bsrn import parse_bsrn  # noqa: F401
 from pvlib.iotools.sodapro import get_cams  # noqa: F401
 from pvlib.iotools.sodapro import read_cams  # noqa: F401
 from pvlib.iotools.sodapro import parse_cams  # noqa: F401

pvlib/iotools/bsrn.py

@@ -3,6 +3,7 @@
 import warnings
 
 import pandas as pd
+import os
 from pkg_resources import parse_version
 import pytest
 from functools import wraps
@@ -82,6 +83,18 @@ def assert_frame_equal(left, right, **kwargs):
                                   reason='does not run on windows')
 
 
+try:
+    # Attempt to load BSRN credentials used for testing pvlib.iotools.get_bsrn
+    bsrn_username = os.environ["BSRN_FTP_USERNAME"]
+    bsrn_password = os.environ["BSRN_FTP_PASSWORD"]
+    has_bsrn_credentials = True
+except KeyError:
+    has_bsrn_credentials = False
+
+requires_bsrn_credentials = pytest.mark.skipif(
+    not has_bsrn_credentials, reason='requires bsrn credentials')
+
+
 try:
     import statsmodels  # noqa: F401
     has_statsmodels = True

pvlib/tests/conftest.py

@@ -2,25 +2,123 @@
 tests for :mod:`pvlib.iotools.bsrn`
 """
 
-
 import pandas as pd
 import pytest
+import os
+from pvlib.iotools import read_bsrn, get_bsrn
+from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal,
+                        requires_bsrn_credentials)
+
+
+@pytest.fixture(scope="module")
+def bsrn_credentials():
+    """Supplies the BSRN FTP credentials for testing purposes.
 
-from pvlib.iotools import bsrn
-from ..conftest import DATA_DIR, assert_index_equal
+    Users should obtain their own credentials as described in the `read_bsrn`
+    documentation."""
+    bsrn_username = os.environ["BSRN_FTP_USERNAME"]
+    bsrn_password = os.environ["BSRN_FTP_PASSWORD"]
+    return bsrn_username, bsrn_password
 
 
-@pytest.mark.parametrize('testfile,expected_index', [
-    ('bsrn-pay0616.dat.gz',
-     pd.date_range(start='20160601', periods=43200, freq='1min', tz='UTC')),
-    ('bsrn-lr0100-pay0616.dat',
-     pd.date_range(start='20160601', periods=43200, freq='1min', tz='UTC')),
+@pytest.fixture
+def expected_index():
+    return pd.date_range(start='20160601', periods=43200, freq='1min',
+                         tz='UTC')
+
+
+@pytest.mark.parametrize('testfile', [
+    ('bsrn-pay0616.dat.gz'),
+    ('bsrn-lr0100-pay0616.dat'),
 ])
 def test_read_bsrn(testfile, expected_index):
-    data = bsrn.read_bsrn(DATA_DIR / testfile)
+    data, metadata = read_bsrn(DATA_DIR / testfile)
+    assert_index_equal(expected_index, data.index)
+    assert 'ghi' in data.columns
+    assert 'dni_std' in data.columns
+    assert 'dhi_min' in data.columns
+    assert 'lwd_max' in data.columns
+    assert 'relative_humidity' in data.columns
+
+
+def test_read_bsrn_logical_records(expected_index):
+    # Test if logical records 0300 and 0500 are correct parsed
+    # and that 0100 is not passed when not specified
+    data, metadata = read_bsrn(DATA_DIR / 'bsrn-pay0616.dat.gz',
+                               logical_records=['0300', '0500'])
+    assert_index_equal(expected_index, data.index)
+    assert 'lwu' in data.columns
+    assert 'uva_global' in data.columns
+    assert 'uvb_reflected_std' in data.columns
+    assert 'ghi' not in data.columns
+
+
+def test_read_bsrn_bad_logical_record():
+    # Test if ValueError is raised if an unsupported logical record is passed
+    with pytest.raises(ValueError, match='not in'):
+        read_bsrn(DATA_DIR / 'bsrn-lr0100-pay0616.dat',
+                  logical_records=['dummy'])
+
+
+def test_read_bsrn_logical_records_not_found():
+    # Test if an empty dataframe is returned if specified LRs are not present
+    data, metadata = read_bsrn(DATA_DIR / 'bsrn-lr0100-pay0616.dat',
+                               logical_records=['0300', '0500'])
+    assert data.empty  # assert that the dataframe is empty
+    assert 'uva_global' in data.columns
+    assert 'uvb_reflected_std' in data.columns
+    assert 'uva_global_max' in data.columns
+    assert 'dni' not in data.columns
+    assert 'day' not in data.columns
+
+
+@requires_bsrn_credentials
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_bsrn(expected_index, bsrn_credentials):
+    # Retrieve irradiance data from the BSRN FTP server
+    # the TAM station is chosen due to its small file sizes
+    username, password = bsrn_credentials
+    data, metadata = get_bsrn(
+        start=pd.Timestamp(2016, 6, 1),
+        end=pd.Timestamp(2016, 6, 29),
+        station='tam',
+        username=username,
+        password=password,
+        local_path='')
     assert_index_equal(expected_index, data.index)
     assert 'ghi' in data.columns
     assert 'dni_std' in data.columns
     assert 'dhi_min' in data.columns
     assert 'lwd_max' in data.columns
     assert 'relative_humidity' in data.columns
+
+
+@requires_bsrn_credentials
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_bsrn_bad_station(bsrn_credentials):
+    # Test if KeyError is raised if a bad station name is passed
+    username, password = bsrn_credentials
+    with pytest.raises(KeyError, match='sub-directory does not exist'):
+        get_bsrn(
+            start=pd.Timestamp(2016, 6, 1),
+            end=pd.Timestamp(2016, 6, 29),
+            station='not_a_station_name',
+            username=username,
+            password=password)
+
+
+@requires_bsrn_credentials
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_bsrn_no_files(bsrn_credentials):
+    username, password = bsrn_credentials
+    # Test if Warning is given if no files are found for the entire time frame
+    with pytest.warns(UserWarning, match='No files'):
+        get_bsrn(
+            start=pd.Timestamp(1990, 6, 1),
+            end=pd.Timestamp(1990, 6, 29),
+            station='tam',
+            username=username,
+            password=password)