diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..545b8f8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +# See https://pre-commit.com for more information +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 # Python 3.6 compatible + hooks: + # Python related checks + - id: check-ast + - id: check-builtin-literals + - id: check-docstring-first + - id: name-tests-test + name: Check unit tests start with 'test_' + args: ['--django'] + files: 'test/.*' + # Other checks + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + name: Force line endings to LF + args: ['--fix=lf'] + - id: trailing-whitespace + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: python-check-mock-methods + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + +# Pre-commit CI config, see https://pre-commit.ci/ +ci: + autofix_prs: false + autoupdate_schedule: quarterly diff --git a/docs/setting_up_VM_with_app.md b/docs/setting_up_VM_with_app.md new file mode 100644 index 0000000..254bf0f --- /dev/null +++ b/docs/setting_up_VM_with_app.md @@ -0,0 +1,27 @@ +## Steps to follow to get a VM with monitoring app running on Apache + +To get a 'prototype' of the monitoring app running with Apache, follow these steps: +- create a cloud VM of the type: scientific-linux-7-aq +- continue by selecting sandbox 'testing_personality_2', archetype 'ral-tier1', personality 'apel-data-validation-test' + +Allow 15 minutes after the machine is created, then remember to edit security groups from OpenStack to allow Apache to work. +Then follow these steps from within the machine: +- quattor-fetch && quattor-configure --all +- cd /usr/share/DJANGO_MONITORING_APP/monitoring +- modify the file settings.py, specifically the dict called DATABASES, to include the right credentials and database names +- cd .. +- source venv/bin/activate +- systemctl restart httpd +- sudo chown apache . + +At this point the app should be working, so just get the ip address by writing "hostname -I" within the machine and the app should be already running at that address. + + +## What to do if the app seems to stop working after closing the VM +If the VM is shut down, next time we try to open the app, Apache might give the error message "Unable to open the database file". +If this happens, just follow these steps on the machine: +1. cd /usr/share/DJANGO_MONITORING_APP +2. source venv/bin/activate +3. sudo chown apache . + +Note that step 2 is necessary for step 3 to work and the error message to disappear. diff --git a/docs/what_gets_installed.md b/docs/what_gets_installed.md new file mode 100644 index 0000000..38d7d38 --- /dev/null +++ b/docs/what_gets_installed.md @@ -0,0 +1,23 @@ +For Django to work with apache, it is common to have a venv within the app, where django and djangorestframework get installed. Other packages needed for the app to work are installed by Aquilon outside the venv. + +## Packages installed by Aquilon outside the venv +Following the config file that Aquilon uses, the following are the packages installed: +- `httpd` +- `python3-mod_wsgi` (for apache to work with django) +- `python3-devel` +- `gcc` (needed for dependencies) +- `mariadb` +- `tar` + +## Packages installed within the venv +Within venv, the following are installed through pip: +- `djangorestframework` (3.15.1) +- `pymysql` (1.0.2) (needed for mariadb to work) +- `pandas` (1.1.5) (needed by the app) +- `django` (3.2.25) +- `pytz` (2025.2) + +Note that when the version of the packages is specified, the app would not work with a different version (due to dependencies conflicts). + +Is is also important to note that different types of OS require different packages to be installed. +The above are the packages that allow the app to work on a Rocky8. diff --git a/docs/what_pages_can_be_accessed.md b/docs/what_pages_can_be_accessed.md new file mode 100644 index 0000000..6f83fb7 --- /dev/null +++ b/docs/what_pages_can_be_accessed.md @@ -0,0 +1,22 @@ +## Pages that can be accessed through the app + +The following urls are the ones that can be accessed without passing any parameter: +- http://ip-address/publishing/cloud/ +- http://ip-address/publishing/gridsync/ + +These pages show info for a number of sites, so do not require a site name to be specified within the url. + +The url http://ip-address/publishing/grid/ , instead, should be used together with the name of the site we are looking for. +For example: http://ip-address/publishing/grid/BelGrid-UCL/ +It is not supposed to be used without passing the name of the site. + +The url http://ip-address/publishing/gridsync/ shows a sync table, and it's probably the most important bit of the personality 'apel-data-validation'. +This table contains data related to many sites, specifically number of jobs being published vs in the db, and this number is shown for every (available) month of each site. + +Clicking on any name in the first column (containing site names) allows to access a similar table which only shows the data relative to that site. +This more specific table is such that the first columns shows the months for which we have data (for that site). +Clicking on the month allows to open another table that shows data for that month and site only, divided by submithost. + +The pages accessed through the links can of course be accessed by typing directly the url. For example, if we want data related to the site 'CSCS-LCG2' and month '2013-11', we would type : +http://ip-address/publishing/gridsync/CSCS-LCG2/2013-11/ +However, in this case if there is no data for the month we are looking for, we would get an error. diff --git a/monitoring/__init__.py b/monitoring/__init__.py index e69de29..063cd2c 100644 --- a/monitoring/__init__.py +++ b/monitoring/__init__.py @@ -0,0 +1,2 @@ +import pymysql +pymysql.install_as_MySQLdb() diff --git a/monitoring/availability/apps.py b/monitoring/availability/apps.py index c7eacd8..53f90fa 100644 --- a/monitoring/availability/apps.py +++ b/monitoring/availability/apps.py @@ -5,4 +5,4 @@ class AvailabilityConfig(AppConfig): - name = 'availability' + name = 'monitoring.availability' diff --git a/monitoring/availability/templates/status.html b/monitoring/availability/templates/status.html new file mode 100644 index 0000000..9b504ca --- /dev/null +++ b/monitoring/availability/templates/status.html @@ -0,0 +1 @@ +{{ message }} diff --git a/monitoring/availability/urls.py b/monitoring/availability/urls.py index 0c9a403..077876e 100644 --- a/monitoring/availability/urls.py +++ b/monitoring/availability/urls.py @@ -1,7 +1,7 @@ -from django.conf.urls import url +from django.urls import path -import views +from monitoring.availability import views urlpatterns = [ - url(r'^$', views.status), + path('', views.status, name='availability'), ] diff --git a/monitoring/availability/views.py b/monitoring/availability/views.py index 380575e..eff3094 100644 --- a/monitoring/availability/views.py +++ b/monitoring/availability/views.py @@ -1,15 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import time - from rest_framework.decorators import api_view from rest_framework.response import Response @api_view() def status(requst): - if int(time.time()) % 2: - return Response("Everything OK") - else: - return Response("Everything NOT ok.") + return Response({"message": "OK"}, status=200, template_name="status.html") diff --git a/monitoring/benchmarks/__init__.py b/monitoring/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/benchmarks/admin.py b/monitoring/benchmarks/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/monitoring/benchmarks/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/monitoring/benchmarks/apps.py b/monitoring/benchmarks/apps.py new file mode 100644 index 0000000..bb0e0b3 --- /dev/null +++ b/monitoring/benchmarks/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class BenchmarksConfig(AppConfig): + name = 'monitoring.benchmarks' diff --git a/monitoring/benchmarks/migrations/__init__.py b/monitoring/benchmarks/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/benchmarks/models.py b/monitoring/benchmarks/models.py new file mode 100644 index 0000000..72f9f64 --- /dev/null +++ b/monitoring/benchmarks/models.py @@ -0,0 +1,54 @@ +from django.db import models + + +class BenchmarksBySubmithost(models.Model): + fetched = models.DateTimeField(auto_now=True) + SiteName = models.CharField(max_length=255) + SubmitHost = models.CharField(max_length=255) + BenchmarkType = models.CharField(max_length=50) + BenchmarkValue = models.DecimalField(max_digits=10, decimal_places=3) + RecordType = models.CharField(max_length=50) + UpdateTime = models.DateTimeField() + + class Meta: + ordering = ('SiteName',) + + +class VJobRecords(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + SubmitHost = models.CharField(max_length=255) + ServiceLevelType = models.CharField(max_length=50) + ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) + UpdateTime = models.DateTimeField() + EndTime = models.DateTimeField() + + class Meta: + managed = False + db_table = 'VJobRecords' + verbose_name = 'Job Record' + + +class VSummaries(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + SubmitHost = models.CharField(max_length=255) + ServiceLevelType = models.CharField(max_length=50) + ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) + UpdateTime = models.DateTimeField() + + class Meta: + managed = False + db_table = 'VSummaries' + verbose_name = 'Summary' + + +class VNormalisedSummaries(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + SubmitHost = models.CharField(max_length=255) + ServiceLevelType = models.CharField(max_length=50) + ServiceLevel = models.DecimalField(max_digits=10, decimal_places=3) + UpdateTime = models.DateTimeField() + + class Meta: + managed = False + db_table = 'VNormalisedSummaries' + verbose_name = 'Normalised Summary' diff --git a/monitoring/benchmarks/serializers.py b/monitoring/benchmarks/serializers.py new file mode 100644 index 0000000..e7eb7cc --- /dev/null +++ b/monitoring/benchmarks/serializers.py @@ -0,0 +1,21 @@ +from rest_framework import serializers + +from monitoring.benchmarks.models import BenchmarksBySubmithost + + +class BenchmarksBySubmithostSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + UpdateTime = serializers.DateTimeField(format=None) + + class Meta: + model = BenchmarksBySubmithost + fields = ( + 'SiteName', + 'SubmitHost', + 'BenchmarkType', + 'BenchmarkValue', + 'RecordType', + 'UpdateTime', + ) diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html new file mode 100644 index 0000000..f2e3f81 --- /dev/null +++ b/monitoring/benchmarks/templates/benchmarks_by_submithost.html @@ -0,0 +1,49 @@ + + + + + + Sites publishing benchmark records + + + +

Sites publishing benchmark records in last 3 months

+

Page last updated: {{ last_fetched|date:"Y-m-d H:i:s.u" }}

+ + + + + + {% for item in site_counts_by_record_type %} + + + + + {% endfor %} +
Record typeNumber of Sites
{{ item.RecordType }}{{ item.site_count }}
+ +
+ + + + + + + + + + + {% for benchmark in benchmarks %} + + + + + + + + + {% endfor %} +
SiteSubmit hostBenchmark typeBenchmark valueRecord typeLast updated
{{ benchmark.SiteName }}{{ benchmark.SubmitHost }}{{ benchmark.BenchmarkType }}{{ benchmark.BenchmarkValue }}{{ benchmark.RecordType }}{{ benchmark.UpdateTime|date:"Y-m-d H:i:s" }}
+ + + diff --git a/monitoring/benchmarks/tests.py b/monitoring/benchmarks/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/monitoring/benchmarks/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/monitoring/benchmarks/urls.py b/monitoring/benchmarks/urls.py new file mode 100644 index 0000000..e3b06cb --- /dev/null +++ b/monitoring/benchmarks/urls.py @@ -0,0 +1,8 @@ +from rest_framework import routers + +from monitoring.benchmarks import views + +router = routers.SimpleRouter() +router.register('', views.BenchmarksViewSet) + +urlpatterns = router.urls diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py new file mode 100644 index 0000000..736d6ba --- /dev/null +++ b/monitoring/benchmarks/views.py @@ -0,0 +1,45 @@ +from django.shortcuts import render +from datetime import datetime, timedelta + +from django.db.models import Max, Count +from django.db.models.functions import Lower + +from rest_framework import viewsets +from rest_framework.renderers import TemplateHTMLRenderer + + +from monitoring.benchmarks.models import BenchmarksBySubmithost + +from monitoring.benchmarks.serializers import BenchmarksBySubmithostSerializer + +class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet): + # Lower('SiteName'): sorts sites alphabetically, case-insensitively. + # '-UpdateTime': sorts records within each site by UpdateTime in descending order (latest first). + queryset = BenchmarksBySubmithost.objects.all().order_by(Lower('SiteName'), '-UpdateTime') + + serializer_class = BenchmarksBySubmithostSerializer + template_name = 'benchmarks_by_submithost.html' + + def list(self, request): + last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max'] + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + response = super(BenchmarksViewSet, self).list(request) + + # Count number of distinct sites per RecordType + site_counts_by_record_type = ( + BenchmarksBySubmithost.objects + .values('RecordType') + .annotate(site_count=Count('SiteName', distinct=True)) + .order_by('RecordType') + ) + + if type(request.accepted_renderer) is TemplateHTMLRenderer: + response.data = { + 'benchmarks': response.data, + 'last_fetched': last_fetched, + 'site_counts_by_record_type': site_counts_by_record_type + } + + return response diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py new file mode 100644 index 0000000..2fc2a77 --- /dev/null +++ b/monitoring/db_update_sqlite.py @@ -0,0 +1,322 @@ +# -*- coding: utf-8 -*- +""" +`db_update_sqlite.py` - Syncs data from external database into local SQLite DB. + - It will be run as a standalone operation via cron. +""" +import configparser +import logging +import os +import sys + +import django +from django.db import DatabaseError +import pandas as pd +from django.utils.timezone import make_aware, is_naive + + +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + +# Find the root and the Django project +sys.path.append(BASE_DIR) + +# Set up Django settings to run this `db_update_sqlite.py` as standalone file +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitoring.settings") + +# Initialize and setup Django +django.setup() + + +# Cron jobs run in a minimal environment and lack access to Django settings. +# To ensure proper model imports and database interactions, we MUST initialize and setup Django first. +from monitoring.publishing.models import ( + GridSite, + CloudSite, + GridSiteSync, + VAnonCloudRecord, + VSuperSummaries, + VSyncRecords +) + +from monitoring.publishing.views import ( + summaries_dict_standard, + syncrecords_dict_standard, + correct_dict, + fill_summaries_dict, + fill_syncrecords_dict, + get_year_month_str +) + +from monitoring.benchmarks.models import ( + BenchmarksBySubmithost, + VJobRecords, + VSummaries, + VNormalisedSummaries, +) + +try: + # Read configuration from the file + cp = configparser.ConfigParser(interpolation=None) + file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') + cp.read(file_path) + +except (configparser.NoSectionError) as err: + print("Error in configuration file. Check that file exists first: %s" % err) + sys.exit(1) + +# Set up basic logging config +logging.basicConfig( + filename=cp.get('common', 'logfile'), + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) + +# set up the logger +log = logging.getLogger(__name__) + + +def determine_sync_status(f): + """ + Helper to determine sync status between published and the database record counts. + """ + RecordCountPublished = f.get("RecordCountPublished") + RecordCountInDb = f.get("RecordCountInDb") + rel_diff1 = abs(RecordCountPublished - RecordCountInDb)/RecordCountInDb + rel_diff2 = abs(RecordCountPublished - RecordCountInDb)/RecordCountPublished + if rel_diff1 < 0.01 or rel_diff2 < 0.01: + syncstatus = "OK" + else: + syncstatus = "ERROR [ Please use the Gap Publisher to synchronise this dataset]" + return syncstatus + + +def refresh_gridsite(): + try: + sql_query = """ + SELECT + Site, + max(LatestEndTime) AS LatestPublish + FROM VSuperSummaries + WHERE LatestEndTime > DATE_SUB(NOW(), INTERVAL 1 YEAR) + GROUP BY 1; + """ + fetchset = VSuperSummaries.objects.using('grid').raw(sql_query) + + for f in fetchset: + GridSite.objects.update_or_create( + defaults={'updated': f.LatestPublish}, + SiteName=f.Site + ) + + log.info("Refreshed GridSite") + + except DatabaseError: + log.exception('Error while trying to refresh GridSite') + + +def refresh_cloudsite(): + try: + sql_query = """ + SELECT + b.SiteName, + COUNT(DISTINCT VMUUID) as VMs, + CloudType, + b.UpdateTime + FROM( + SELECT + SiteName, + MAX(UpdateTime) AS latest + FROM VAnonCloudRecords + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 1 YEAR) + GROUP BY SiteName + ) + AS a + INNER JOIN VAnonCloudRecords + AS b + ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest + GROUP BY SiteName; + """ + fetchset = VAnonCloudRecord.objects.using('cloud').raw(sql_query) + + for f in fetchset: + CloudSite.objects.update_or_create( + defaults={ + 'Vms': f.VMs, + 'Script': f.CloudType, + 'updated': f.UpdateTime + }, + SiteName=f.SiteName + ) + + log.info("Refreshed CloudSite") + + except DatabaseError: + log.exception('Error while trying to refresh CloudSite') + + +def refresh_gridsitesync(): + try: + # The condition on EarliestEndTime and LatestEndTime is necessary to avoid error by pytz because of dates like '00-00-00' + sql_query_summaries = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountPublished, + MIN(EarliestEndTime) AS RecordStart, + MAX(LatestEndTime) AS RecordEnd + FROM VSuperSummaries + WHERE + Year >= YEAR(NOW()) - 2 AND + EarliestEndTime>'1900-01-01' AND + LatestEndTime>'1900-01-01' + GROUP BY + Site, Year, Month; + """ + fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) + + sql_query_syncrec = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountInDb + FROM VSyncRecords + WHERE + Year >= YEAR(NOW()) - 2 + GROUP BY + Site, Year, Month; + """ + fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrec) + + # Create empty dicts that will become dfs to be combined + summaries_dict = summaries_dict_standard.copy() + syncrecords_dict = syncrecords_dict_standard.copy() + + # Fill the dicts with the fetched data + for row in fetchset_Summaries: + summaries_dict = fill_summaries_dict(summaries_dict, row) + summaries_dict = correct_dict(summaries_dict) + for row in fetchset_SyncRecords: + syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) + syncrecords_dict = correct_dict(syncrecords_dict) + + # Merge data from VSuperSummaries and VSyncRecords into one df + df_Summaries = pd.DataFrame.from_dict(summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) + df_all = df_Summaries.merge( + df_SyncRecords, + left_on=['Site', 'Month', 'Year'], + right_on=['Site', 'Month', 'Year'], + how='inner' + ) + fetchset = df_all.to_dict('index') + + # Determine SyncStatus based on the difference between records published and in db + for f in fetchset.values(): + f['SyncStatus'] = determine_sync_status(f) + + # Combined primary keys outside the default dict + GridSiteSync.objects.update_or_create( + defaults={ + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + 'SyncStatus': f.get("SyncStatus"), + }, + YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), + SiteName=f.get("Site"), + Month=f.get("Month"), + Year=f.get("Year"), + ) + log.info("Refreshed GridSiteSync") + + except DatabaseError: + log.exception('Error while trying to refresh GridSiteSync') + + +def refresh_BenchmarksBySubmitHost(): + views = ['VSummaries', 'VJobRecords', 'VNormalisedSummaries'] + for view in views: + refresh_BenchmarksBySubmitHost_from_view(view) + + +def refresh_BenchmarksBySubmitHost_from_view(view_name): + try: + if view_name == 'VSummaries': + sql_query = f""" + SELECT + Site, + SubmitHost, + ServiceLevelType, + ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM {view_name} + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; + """ + elif view_name == 'VJobRecords': + sql_query = f""" + SELECT + Site, + SubmitHost, + ServiceLevelType, + ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM {view_name} + WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + GROUP BY Site, SubmitHost; + """ + elif view_name == 'VNormalisedSummaries': + sql_query = f""" + SELECT + Site, + SubmitHost, + ServiceLevelType, + (NormalisedWallDuration / WallDuration) AS ServiceLevel, + max(UpdateTime) AS LatestPublish + FROM {view_name} + WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH) + AND WallDuration > 0 + GROUP BY Site, SubmitHost; + """ + else: + log.warning(f"Unknown view name: {view_name}") + return + + # Dynamically get the model class from globals + model_class = globals()[view_name] + fetchset = model_class.objects.using('grid').raw(sql_query) + + for f in fetchset: + BenchmarksBySubmithost.objects.update_or_create( + defaults={ + 'UpdateTime': make_aware(f.LatestPublish) if is_naive(f.LatestPublish) else f.LatestPublish, + 'RecordType': model_class._meta.verbose_name + }, + SiteName=f.Site, + SubmitHost=f.SubmitHost, + BenchmarkType=f.ServiceLevelType, + BenchmarkValue=f.ServiceLevel, + ) + + log.info(f"Refreshed BenchmarksBySubmitHost from {view_name}") + + except Exception: + log.exception(f'Error while trying to refresh BenchmarksBySubmitHost from {view_name}') + + +if __name__ == "__main__": + log.info('=====================') + + refresh_gridsite() + refresh_cloudsite() + refresh_gridsitesync() + refresh_BenchmarksBySubmitHost() + + log.info( + "Data retrieval and processing attempted. " + "Check the above logs for details on the sync status" + ) + log.info('=====================') diff --git a/monitoring/publishing/apps.py b/monitoring/publishing/apps.py index 57c17f7..d5b06c3 100644 --- a/monitoring/publishing/apps.py +++ b/monitoring/publishing/apps.py @@ -5,4 +5,4 @@ class PublishingConfig(AppConfig): - name = 'publishing' + name = 'monitoring.publishing' diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py index ba65c9a..195872d 100644 --- a/monitoring/publishing/models.py +++ b/monitoring/publishing/models.py @@ -4,18 +4,67 @@ from django.db import models +class GridSite(models.Model): + fetched = models.DateTimeField(auto_now=True) + SiteName = models.CharField(max_length=255, primary_key=True) + updated = models.DateTimeField() + + +class VSuperSummaries(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + LatestPublish = models.DateTimeField() + Month = models.IntegerField() + Year = models.IntegerField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() + RecordCountPublished = models.IntegerField() + + class Meta: + managed = False + db_table = 'VSuperSummaries' + + +class GridSiteSync(models.Model): + fetched = models.DateTimeField(auto_now=True) + SiteName = models.CharField(max_length=255) + YearMonth = models.CharField(max_length=255) + Year = models.IntegerField() + Month = models.IntegerField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() + RecordCountPublished = models.IntegerField() + RecordCountInDb = models.IntegerField() + SyncStatus = models.CharField(max_length=255) + + class Meta: + # Descending order of Year and Month to display latest data first + ordering = ('SiteName', '-Year', '-Month') + unique_together = ('SiteName', 'YearMonth') + + +class VSyncRecords(models.Model): + Site = models.CharField(max_length=255, primary_key=True) + RecordCountInDb = models.IntegerField() + + class Meta: + managed = False + db_table = 'VSyncRecords' + + class CloudSite(models.Model): fetched = models.DateTimeField(auto_now=True) - name = models.CharField(max_length=255, primary_key=True) - script = models.CharField(max_length=255) + SiteName = models.CharField(max_length=255, primary_key=True) + Vms = models.IntegerField(default=0) + Script = models.CharField(max_length=255) updated = models.DateTimeField() class Meta: - ordering = ('name',) + ordering = ('SiteName',) class VAnonCloudRecord(models.Model): SiteName = models.CharField(max_length=255, primary_key=True) + VMs = models.IntegerField() CloudType = models.CharField(max_length=255) UpdateTime = models.DateTimeField() @@ -24,6 +73,25 @@ class Meta: db_table = 'vanoncloudrecords' def __str__(self): - return '%s, running "%s", updated at %s' % (self.SiteName, + return '%s running "%s" updated at %s with %s records' % ( + self.SiteName, self.CloudType, - self.UpdateTime) + self.UpdateTime, + self.VMs) + + +class GridSiteSyncSubmitH(models.Model): + fetched = models.DateTimeField(auto_now=True) + SiteName = models.CharField(max_length=255) + YearMonth = models.CharField(max_length=255) + Year = models.IntegerField() + Month = models.IntegerField() + RecordStart = models.DateTimeField() + RecordEnd = models.DateTimeField() + RecordCountPublished = models.IntegerField() + RecordCountInDb = models.IntegerField() + SubmitHost = models.CharField(max_length=255) + + class Meta: + ordering = ('SiteName', '-Year', '-Month') + unique_together = ('SiteName', 'YearMonth', 'SubmitHost') diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py index 40309bb..b273462 100644 --- a/monitoring/publishing/serializers.py +++ b/monitoring/publishing/serializers.py @@ -1,8 +1,119 @@ from rest_framework import serializers +from rest_framework.reverse import reverse + +from monitoring.publishing.models import ( + CloudSite, + GridSite, + GridSiteSync, + GridSiteSyncSubmitH +) + + +class GridSiteSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + updated = serializers.DateTimeField(format=None) + + class Meta: + model = GridSite + fields = ( + 'url', + 'SiteName', + 'updated' + ) + + # Sitename substitutes for pk + extra_kwargs = { + 'url': {'view_name': 'gridsite-detail', 'lookup_field': 'SiteName'} + } + + +class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + + class Meta: + model = GridSiteSync + fields = ( + 'url', + 'SiteName', + 'YearMonth', + 'RecordStart', + 'RecordEnd', + 'RecordCountPublished', + 'RecordCountInDb', + 'SyncStatus' + ) + + # Sitename substitutes for pk + extra_kwargs = { + 'url': {'view_name': 'gridsitesync-detail', 'lookup_field': 'SiteName'} + } -from models import CloudSite class CloudSiteSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + updated = serializers.DateTimeField(format=None) + class Meta: model = CloudSite - fields = ('url', 'name', 'script', 'updated') + fields = ( + 'url', + 'SiteName', + 'Vms', + 'Script', + 'updated' + ) + + # Sitename substitutes for pk + extra_kwargs = { + 'url': {'view_name': 'cloudsite-detail', 'lookup_field': 'SiteName'} + } + + +class MultipleFieldLookup(serializers.HyperlinkedIdentityField): + # HyperlinkedModelSerializer seems to NOT able to work with two lookup_fields + # This class is ONLY capable to match object instance to its URL representation. + # i.e, `SiteName` and `YearMonth` ONLY + # + # Overriding the get_url() method - To match object instance to its URL representation. + def get_url(self, obj, view_name, request, format): + if not obj.SiteName or not obj.YearMonth: + return None + + return request.build_absolute_uri( + reverse( + view_name, + kwargs={ + 'SiteName': obj.SiteName, + 'YearMonth': obj.YearMonth + }, + request=request, + format=format + )) + + +class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + + # This helps us to match or construct the absolute URL based on the `SiteName` and `YearMonth` + url = MultipleFieldLookup(view_name='gridsync-submithost') + + class Meta: + model = GridSiteSyncSubmitH + fields = ( + 'url', + 'SiteName', + 'YearMonth', + 'RecordStart', + 'RecordEnd', + 'RecordCountPublished', + 'RecordCountInDb', + 'SubmitHost' + ) diff --git a/monitoring/publishing/static/style.css b/monitoring/publishing/static/style.css new file mode 100644 index 0000000..85a5cf9 --- /dev/null +++ b/monitoring/publishing/static/style.css @@ -0,0 +1,53 @@ +/*- Menu Tabs E--------------------------- */ + + #tabsE { + float:left; + width:100%; + background:#333; + font-size:93%; + line-height:normal; + + } + #tabsE ul { + margin:0; + padding:10px 10px 0 50px; + list-style:none; + } + #tabsE li { + display:inline; + margin:0; + padding:0; + } + #tabsE a { + float:left; + background:url("tableftE.gif") no-repeat left top; + margin:0; + padding:0 0 0 4px; + text-decoration:none; + } + #tabsE a span { + float:left; + display:block; + background:url("tabrightE.gif") no-repeat right top; + padding:5px 15px 4px 6px; + color:#fff; + } + /* Commented Backslash Hack hides rule from IE5-Mac \*/ + #tabsE a span {float:none;} + /* End IE5-Mac hack */ + #tabsE a:hover span { + color:#FFF; + } + #tabsE a:hover { + background-position:0% -42px; + } + #tabsE a:hover span { + background-position:100% -42px; + } + + #tabsE #current a { + background-position:0% -42px; + } + #tabsE #current a span { + background-position:100% -42px; + } diff --git a/monitoring/publishing/static/stylesheet.css b/monitoring/publishing/static/stylesheet.css new file mode 100644 index 0000000..11ce5f0 --- /dev/null +++ b/monitoring/publishing/static/stylesheet.css @@ -0,0 +1,184 @@ +h12 { + font-size: 22px; + color: #336699; + background-color: #FFFFFF; + font-family: Arial, Helvetica, sans-serif; + font-weight: bold +} +h1 { + font-size: 1.2em; + color: #336699; + background-color: #FFFFFF; + font-family: Arial, Helvetica, sans-serif; +} +h2 { + font-size: 1em; +} + + +h6 { + align: right; + font-size: 0.6em; +} +body { + font-family: Arial, Helvetica, sans-serif; + background-color: #FFFFFF; + color: #000000; + font-size: .5px; + font-size: 14px; +} +th { + font-family: Arial, Helvetica, sans-serif; + font-size: 12px; + color: #009999; + background-color: #FFFFFF; + font-weight: bold; + align: left; +} +a:link { + text-decoration: none; + color: #336699; + font-weight: bold; +} +a:active { + text-decoration: none; + color: #336699; + font-weight: bold; + background-color: #FFFFFF; +} +a:visited { + text-decoration: none; + color: #996699; + font-weight: bold; +} +a:hover { + color : #666666; + background-color: #CCCCCC; + font-weight: bold; +} +hr { + color: #666666; + background-color: #FFFFFF; +} +.outlined { + border: 1px solid #000000; +} +.navbar-title { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #FFFFFF; + font-weight: bold; + + +} +.navbar { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.6em; + background-color: #FFFFFF; + +} +p { + font-family: Arial, Helvetica, sans-serif; + color: #000000; + font-size: 14px; + font-weight: normal; +} +li { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #111111; + list-style-type: square; + font-weight: normal; +} +.sidebar-orange { + background-color: #ffe5b2; + background-image: url(images/orange-globe.jpg); + background-position: right center; + background-repeat: no-repeat; + +} +.sidebar-green { + background-image: url(images/green-pulse.jpg); + background-repeat: repeat-x; + background-position: center bottom; + background-color: #dcf6de; +} +.sidebar-blue { + background-color: #dcf4f6; + background-image: url(images/blue-bars.jpg); + background-repeat: no-repeat; + background-position: right bottom; +} +.sidebar-pink { + background-color: #efdcf6; + background-image: url(images/pink-news.jpg); + background-repeat: no-repeat; + background-position: left bottom; +} +.note { + background-color: #E4E4E4; + border: 1px dotted #000000; + padding: 5px; +} +.tabletext { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #000000; + background-color: #DDDDDD; +} +.tabletextwarning { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #000000; + background-color: #FFFF00; +} +.tabletextok { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #000000; + background-color: #00FF00; +} +.tabletexterror { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #000000; + background-color: #FF0000; +} +.tabletextinfo { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #000000; + background-color: #00CCFF; +} +.tableheader { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + color: #FFFFFF; + background-color: #000000; + font-weight: bold; + text-align: center; +} +.navbar-heading { + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + background-color: #FFFFFF; + font-weight: bold; + color: #000000; +} +.feintoutlined { + border: 1px dotted #CCCCCC; +} +.newsHeader { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.9em; + background-color: #97A6CD; + font-weight: bold; +} +.newsBody { + font-family: Arial, Helvetica, sans-serif; + font-size: 0.7em; + background-color: #ACBBDA; +} +.invisibleBorder { + border: thin dashed #EEEEEE; +} diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html index e146485..0ffc1ef 100644 --- a/monitoring/publishing/templates/cloudsites.html +++ b/monitoring/publishing/templates/cloudsites.html @@ -1,19 +1,23 @@ + + + Sites publishing cloud accounting records -

Sites publishing cloud accounting records from 2018-06-19 onwards

-

Page last updated: {{ last_fetched|date:"c" }}

+

Sites publishing cloud accounting records in the last year

+

Data last fetched: {{ last_fetched|date:"c" }}

- + {% for site in sites %} - - - + + + + {% endfor %}
SiteCloudTypeLastUpdatedSiteVMsInLastUpdateCloudTypeLastUpdated
{{ site.name }}{{ site.script }}{{ site.updated }}{{ site.SiteName }}{{ site.Vms }}{{ site.Script }}{{ site.updated|date:"Y-m-d H:i:s" }}
diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html new file mode 100644 index 0000000..dc5426e --- /dev/null +++ b/monitoring/publishing/templates/gridsites.html @@ -0,0 +1,39 @@ +{% load static %} + + + + + + APEL Publication Summary + + + + +

APEL Publication Test

+ + +
+ + + + + + + + + {% for site in sites %} + + + + + + + {% endfor %} +
SiteMeasurement DateMeasurement TimePublication Status
{{ site.SiteName }}{{ site.updated|date:"Y-m-d" }}{{ site.updated|date:"G:i:s" }}{{ site.stdout }}
+ + diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html new file mode 100644 index 0000000..8c49d4e --- /dev/null +++ b/monitoring/publishing/templates/gridsync.html @@ -0,0 +1,48 @@ +{% load static %} +{% load humanize %} + + + + + + APEL Publication Summary + + + + +

APEL Synchronisation Test

+ + +
+ + + + + + + + + + + + + + {% for record in records.results %} + + + + + + + + + + {% endfor %} +
All sites
Site NameMonthRecord StartRecord EndRecord Count
In Your Database		Record Count
What You Published		Synchronisation Status
{{ record.SiteName }}{{ record.YearMonth }}{{ record.RecordStart }}{{ record.RecordEnd }}{{ record.RecordCountPublished|intcomma }}{{ record.RecordCountInDb|intcomma }}{{ record.SyncStatus }}
+ + diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html new file mode 100644 index 0000000..d332194 --- /dev/null +++ b/monitoring/publishing/templates/gridsync_singlesite.html @@ -0,0 +1,48 @@ +{% load static %} +{% load humanize %} + + + + + + APEL Publication Summary + + + + +

APEL Synchronisation Test

+ + +
+ + + + + + + + + + + + + {% for record in records %} + + + + + + + + + {% endfor %} +
{{ records.0.SiteName }}
MonthRecord StartRecord EndRecord Count
In Your Database		Record Count
What You Published		Synchronisation Status
+ {{ record.YearMonth }} + {{ record.RecordStart }}{{ record.RecordEnd }}{{ record.RecordCountPublished|intcomma }}{{ record.RecordCountInDb|intcomma }}{{ record.SyncStatus }}
+ + diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html new file mode 100644 index 0000000..28f1dbb --- /dev/null +++ b/monitoring/publishing/templates/gridsync_submithost.html @@ -0,0 +1,46 @@ +{% load static %} +{% load humanize %} + + + + + + APEL Publication Summary + + + + +

APEL Synchronisation Test

+ + +
+ + + + + + + + + + + + + {% for host in submisthosts %} + + + + + + + + + {% endfor %} +
{{ submisthosts.0.SiteName }}, {{ submisthosts.0.YearMonth }}
MonthSubmitHostRecord StartRecord EndRecord Count
In Your Database		Record Count
What You Published
{{ host.YearMonth }} {{ host.SubmitHost }}{{ host.RecordStart }}{{ host.RecordEnd }}{{ host.RecordCountPublished|intcomma }}{{ host.RecordCountInDb|intcomma }}
+ + diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py index c45044e..cf0f642 100644 --- a/monitoring/publishing/urls.py +++ b/monitoring/publishing/urls.py @@ -1,13 +1,34 @@ -from django.conf.urls import include, url - from rest_framework import routers -import views +from monitoring.publishing import views +from django.urls import re_path router = routers.SimpleRouter() router.register(r'cloud', views.CloudSiteViewSet) - +router.register(r'grid', views.GridSiteViewSet) +router.register(r'gridsync', views.GridSiteSyncViewSet) urlpatterns = [ - url(r'^', include(router.urls)), + re_path( + r'^cloud/(?P[a-zA-Z0-9._-]+)/$', + views.CloudSiteViewSet.as_view({'get': 'retrieve'}), + name='cloudsite-detail' + ), + re_path( + r'^grid/(?P[a-zA-Z0-9._-]+)/$', + views.GridSiteViewSet.as_view({'get': 'retrieve'}), + name='gridsite-detail' + ), + re_path( + r'^gridsync/(?P[a-zA-Z0-9._-]+)/$', + views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}), + name='gridsitesync-detail' + ), + re_path( + r'^gridsync/(?P[a-zA-Z0-9._-]+)/(?P[0-9-]+)/$', + views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}), + name='gridsync-submithost' + ), ] + +urlpatterns += router.urls diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py index 81d1656..0e07e74 100644 --- a/monitoring/publishing/views.py +++ b/monitoring/publishing/views.py @@ -4,32 +4,381 @@ from datetime import datetime, timedelta from django.db.models import Max +from django.shortcuts import get_object_or_404 +import pandas as pd from rest_framework import viewsets from rest_framework.renderers import TemplateHTMLRenderer +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination -from models import CloudSite, VAnonCloudRecord -from serializers import CloudSiteSerializer +from monitoring.publishing.models import ( + GridSite, + VSuperSummaries, + CloudSite, + GridSiteSync, + VSyncRecords, + GridSiteSyncSubmitH +) -class CloudSiteViewSet(viewsets.ModelViewSet): +from monitoring.publishing.serializers import ( + GridSiteSerializer, + CloudSiteSerializer, + GridSiteSyncSerializer, + GridSiteSyncSubmitHSerializer +) + +summaries_dict_standard = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountPublished": [], + "RecordStart": [], + "RecordEnd": [], + "SubmitHostSumm": [], +} + +syncrecords_dict_standard = { + "Site": [], + "Month": [], + "Year": [], + "RecordCountInDb": [], + "SubmitHostSync": [] +} + + +def update_dict_stdout_and_returncode(single_dict, date): + diff = datetime.today() - date + date = date.strftime("%Y-%m-%d") + + if diff <= timedelta(days=7): + single_dict['returncode'] = 0 + single_dict['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date) + elif diff > timedelta(days=7): + single_dict['returncode'] = 1 + single_dict['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date) + else: + single_dict['returncode'] = 3 + single_dict['stdout'] = "UNKNOWN" + return single_dict + + +def fill_summaries_dict(inpDict, row): + + fields_to_update_and_value_to_add = { + "Site": row.Site, + "Month": row.Month, + "Year": row.Year, + "RecordCountPublished": row.RecordCountPublished, + "RecordStart": row.RecordStart, + "RecordEnd": row.RecordEnd, + } + + for field, value in fields_to_update_and_value_to_add.items(): + inpDict[field] = inpDict.get(field) + [value] + + if hasattr(row, "SubmitHostSumm"): + inpDict["SubmitHostSumm"] = inpDict.get("SubmitHostSumm") + [row.SubmitHostSumm] + + return inpDict + + +def fill_syncrecords_dict(inpDict, row): + inpDict["Site"] = inpDict.get("Site") + [row.Site] + inpDict["Month"] = inpDict.get("Month") + [row.Month] + inpDict["Year"] = inpDict.get("Year") + [row.Year] + inpDict["RecordCountInDb"] = inpDict.get("RecordCountInDb") + [row.RecordCountInDb] + if hasattr(row, "SubmitHostSync"): + inpDict["SubmitHostSync"] = inpDict.get("SubmitHostSync") + [row.SubmitHostSync] + return inpDict + + +def correct_dict(inpDict): + keys_to_remove = [] + for key, val in inpDict.items(): + if len(val) == 0: + keys_to_remove.append(key) + for key in keys_to_remove: + inpDict.pop(key) + return inpDict + + +# Combine Year and Month into one string (display purposes) +def get_year_month_str(year, month): + year_string = str(year) + month_string = str(month) + if len(month_string) == 1: + month_string = '0' + month_string + return year_string + '-' + month_string + + +class GridSiteViewSet(viewsets.ReadOnlyModelViewSet): + queryset = GridSite.objects.all() + serializer_class = GridSiteSerializer + template_name = 'gridsites.html' + lookup_field = 'SiteName' + + def list(self, request): + last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + final_response = [] + response = super(GridSiteViewSet, self).list(request) + + for single_dict in response.data: + date = single_dict.get('updated').replace(tzinfo=None) + single_dict = update_dict_stdout_and_returncode(single_dict, date) + final_response.append(single_dict) + + if type(request.accepted_renderer) is TemplateHTMLRenderer: + response.data = { + 'sites': final_response, + 'last_fetched': last_fetched + } + + return response + + def retrieve(self, request, SiteName=None): + last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max'] + # If there's no data then last_fetched is None. + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + response = super(GridSiteViewSet, self).retrieve(request) + date = response.data['updated'].replace(tzinfo=None) + response.data = update_dict_stdout_and_returncode(response.data, date) + + # Wrap data in a dict so that it can display in template. + if type(request.accepted_renderer) is TemplateHTMLRenderer: + # Single result put in list to work with same HTML template. + response.data = { + 'sites': [response.data], + 'last_fetched': last_fetched + } + + return response + + +class GridSiteSyncPagination(PageNumberPagination): + page_size = 1000 # Number of items to be fetched per page + + +class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet): + queryset = GridSiteSync.objects.all() + serializer_class = GridSiteSyncSerializer + lookup_field = 'SiteName' + pagination_class = GridSiteSyncPagination + + # When a single site is showed (retrieve function used), the template + # is different than the one used when showing a list of sites + def get_template_names(self): + if self.action == 'list': + return ['gridsync.html'] + elif self.action == 'retrieve': + return ['gridsync_singlesite.html'] + + def list(self, request): + last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] + + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + response = super(GridSiteSyncViewSet, self).list(request) + response.data = { + 'records': response.data, + 'last_fetched': last_fetched + } + return response + + def retrieve(self, request, SiteName=None): + last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] + + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + sites_list_qs = GridSiteSync.objects.filter(SiteName=SiteName) + sites_list_serializer = self.get_serializer(sites_list_qs, many=True) + + response = { + 'records': sites_list_serializer.data, + 'last_fetched': last_fetched + } + return Response(response) + + +# Needed for passing two parameters to a viewset (GridSiteSyncSubmitHViewSet) +class MultipleFieldLookupMixin: + """ + Apply this mixin to any view or viewset to get multiple field filtering + based on a `lookup_fields` attribute, instead of the default single field filtering. + """ + def get_object(self): + queryset = self.get_queryset() + queryset = self.filter_queryset(queryset) + filter = {} + for field in self.lookup_fields: + if self.kwargs.get(field): + filter[field] = self.kwargs[field] + obj = get_object_or_404(queryset, **filter) + self.check_object_permissions(self.request, obj) + return obj + + +class GridSiteSyncSubmitHViewSet(MultipleFieldLookupMixin, viewsets.ReadOnlyModelViewSet): + queryset = GridSiteSyncSubmitH.objects.all() + serializer_class = GridSiteSyncSubmitHSerializer + template_name = 'gridsync_submithost.html' + lookup_fields = ('SiteName', 'YearMonth') + + def list(self, request): + last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max'] + response = super(GridSiteSyncSubmitHViewSet, self).list(request) + response.data = { + 'submisthosts': response.data, + 'last_fetched': last_fetched + } + return response + + def retrieve(self, request, SiteName=None, YearMonth=None): + last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max'] + Year, Month = YearMonth.split('-') + sitename_in_table = None + yearmonth_in_table = None + + # This is to ensure the data is updated when changing month + if GridSiteSyncSubmitH.objects.count() > 0: + row_1 = GridSiteSyncSubmitH.objects.filter()[:1].get() + sitename_in_table = row_1.SiteName + yearmonth_in_table = row_1.YearMonth + + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)) or (sitename_in_table != SiteName) or (yearmonth_in_table != YearMonth): + print('Out of date') + + sql_query_summaries = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountPublished, + SubmitHost AS SubmitHostSumm, + MIN(EarliestEndTime) AS RecordStart, + MAX(LatestEndTime) AS RecordEnd + FROM VSuperSummaries + WHERE + Site='{}' AND + Month='{}' AND + Year='{}' + GROUP BY SubmitHost; + """.format(SiteName, Month, Year) + fetchset_Summaries = VSuperSummaries.objects.using('grid').raw(sql_query_summaries) + + sql_query_syncrecords = """ + SELECT + Site, + Month, + Year, + SUM(NumberOfJobs) AS RecordCountInDb, + SubmitHost AS SubmitHostSync + FROM VSyncRecords + WHERE + Site='{}' AND + Month='{}' AND + Year='{}' + GROUP BY SubmitHost; + """.format(SiteName, Month, Year) + fetchset_SyncRecords = VSyncRecords.objects.using('grid').raw(sql_query_syncrecords) + + summaries_dict = summaries_dict_standard.copy() + syncrecords_dict = syncrecords_dict_standard.copy() + + for row in fetchset_Summaries: + summaries_dict = fill_summaries_dict(summaries_dict, row) + summaries_dict = correct_dict(summaries_dict) + for row in fetchset_SyncRecords: + syncrecords_dict = fill_syncrecords_dict(syncrecords_dict, row) + syncrecords_dict = correct_dict(syncrecords_dict) + + df_Summaries = pd.DataFrame.from_dict(summaries_dict) + df_SyncRecords = pd.DataFrame.from_dict(syncrecords_dict) + df_Summaries.dropna(inplace=True) + df_SyncRecords.dropna(inplace=True) + + df_all = df_Summaries.merge( + df_SyncRecords, + left_on=['Site', 'Month', 'Year', 'SubmitHostSumm'], + right_on=['Site', 'Month', 'Year', 'SubmitHostSync'], + how='outer' + ) + + fetchset = df_all.to_dict('index') + + # This is to list only data for one month + GridSiteSyncSubmitH.objects.all().delete() + + for f in fetchset.values(): + GridSiteSyncSubmitH.objects.update_or_create( + defaults={ + 'RecordStart': f.get("RecordStart"), + 'RecordEnd': f.get("RecordEnd"), + 'RecordCountPublished': f.get("RecordCountPublished"), + 'RecordCountInDb': f.get("RecordCountInDb"), + }, + SiteName=f.get("Site"), + YearMonth=get_year_month_str(f.get("Year"), f.get("Month")), + Month=f.get("Month"), + Year=f.get("Year"), + SubmitHost=f.get("SubmitHostSumm"), + ) + + else: + print('No need to update') + + response = super(GridSiteSyncSubmitHViewSet, self).list(request) + response.data = { + 'submisthosts': response.data, + 'last_fetched': last_fetched + } + return response + + +class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet): queryset = CloudSite.objects.all() serializer_class = CloudSiteSerializer template_name = 'cloudsites.html' + lookup_field = 'SiteName' def list(self, request): last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] - print last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20) - if last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): - print 'Out of date' - fetchset = VAnonCloudRecord.objects.using('repository').raw("SELECT b.SiteName, COUNT(DISTINCT VMUUID), CloudType, b.UpdateTime FROM (SELECT SiteName, MAX(UpdateTime) AS latest FROM VAnonCloudRecords WHERE UpdateTime>'2018-07-25' GROUP BY SiteName) AS a INNER JOIN VAnonCloudRecords AS b ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest GROUP BY SiteName") - for f in fetchset: - CloudSite.objects.update_or_create(defaults={'script': f.CloudType, 'updated': f.UpdateTime}, name=f.SiteName) - else: - print 'No need to update' + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) response = super(CloudSiteViewSet, self).list(request) # Wrap data in a dict so that it can display in template. if type(request.accepted_renderer) is TemplateHTMLRenderer: - response.data = {'sites': response.data, 'last_fetched': last_fetched} + response.data = { + 'sites': response.data, + 'last_fetched': last_fetched + } + return response + + def retrieve(self, request, SiteName=None): + last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max'] + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + + response = super(CloudSiteViewSet, self).retrieve(request) + # Wrap data in a dict so that it can display in template. + if type(request.accepted_renderer) is TemplateHTMLRenderer: + # Single result put in list to work with same HTML template. + response.data = { + 'sites': [response.data], + 'last_fetched': last_fetched + } + + response.data['returncode'] = 3 + response.data['stdout'] = "UNKNOWN" + return response diff --git a/monitoring/settings.ini b/monitoring/settings.ini new file mode 100644 index 0000000..fdb8b35 --- /dev/null +++ b/monitoring/settings.ini @@ -0,0 +1,34 @@ +# This file will be parsed by settings.py. + +[common] +# A new key can be generated using django.core.management.utils.get_random_secret_key() +secret_key = + +# `allowed_hosts` values should be comma separated list of hostnames (fqdn's) +allowed_hosts = + +# Path to the log file for `db_update_sqlite.py` execution info. +logfile = + +# Information about the database connection - grid +[db_grid] +# type of database - refers to the Django db backend +backend = django.db.backends.mysql + +hostname = localhost +port = 3306 +name = +username = root +password = + + +# Information about the database connection - cloud +[db_cloud] +# type of database refers to the Django db backend +backend = django.db.backends.mysql + +hostname = localhost +port = 3306 +name = +username = root +password = diff --git a/monitoring/settings.py b/monitoring/settings.py index d76e5d1..65d0352 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -10,23 +10,59 @@ https://docs.djangoproject.com/en/1.11/ref/settings/ """ +import configparser import os +import sys # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'ge^fd9rf)htmxji8kf=jk8frh3=^11@^n=h14gu*fqt^0-lnr$' +# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False + +try: + + # Read configuration from the file + cp = configparser.ConfigParser(interpolation=None) + file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') + cp.read(file_path) + + # SECURITY WARNING: keep the secret key used in production secret! + SECRET_KEY = cp.get('common', 'secret_key') -ALLOWED_HOSTS = [] + ALLOWED_HOSTS = cp.get('common', 'allowed_hosts').split(',') + + # Database + # https://docs.djangoproject.com/en/1.11/ref/settings/#databases + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + }, + 'grid': { + 'ENGINE': cp.get('db_grid', 'backend'), + 'HOST': cp.get('db_grid', 'hostname'), + 'PORT': cp.get('db_grid', 'port'), + 'NAME': cp.get('db_grid', 'name'), + 'USER': cp.get('db_grid', 'username'), + 'PASSWORD': cp.get('db_grid', 'password'), + }, + 'cloud': { + 'ENGINE': cp.get('db_cloud', 'backend'), + 'HOST': cp.get('db_cloud', 'hostname'), + 'PORT': cp.get('db_cloud', 'port'), + 'NAME': cp.get('db_cloud', 'name'), + 'USER': cp.get('db_cloud', 'username'), + 'PASSWORD': cp.get('db_cloud', 'password'), + }, + } +except (configparser.NoSectionError) as err: + print("Error in configuration file. Check that file exists first: %s" % err) + sys.exit(1) # Application definition @@ -34,19 +70,21 @@ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', + 'django.contrib.humanize', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', - 'monitoring.publishing' + 'monitoring.publishing', + 'monitoring.availability', + 'monitoring.benchmarks', ] REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( + 'rest_framework.renderers.TemplateHTMLRenderer', 'rest_framework.renderers.JSONRenderer', 'rest_framework.renderers.BrowsableAPIRenderer', - 'rest_framework.renderers.TemplateHTMLRenderer', - 'rest_framework.renderers.AdminRenderer', ) } @@ -65,7 +103,8 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], + # Add project-wide templates directory + 'DIRS': [os.path.join(BASE_DIR, 'monitoring', 'templates'),], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -81,24 +120,6 @@ WSGI_APPLICATION = 'monitoring.wsgi.application' -# Database -# https://docs.djangoproject.com/en/1.11/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - }, - 'repository': { - 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', - 'PORT': '3306', - 'NAME': 'django_test', - 'USER': 'root', - }, -} - - # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators @@ -136,3 +157,4 @@ # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, "static/") diff --git a/monitoring/templates/home.html b/monitoring/templates/home.html new file mode 100644 index 0000000..f2fa27a --- /dev/null +++ b/monitoring/templates/home.html @@ -0,0 +1,32 @@ + + + + +

APEL Data Validation

+

Welcome. This page gives an overview of the views available in the APEL Data Validation system.

+ +

EGI/WLCG Grid Accounting

+ + +

EGI Cloud Accounting

+

+ +

Return format

+

+ The views above return HTML by default, but can be set to return JSON by adding ?format=json at the end of the URL. + For example: {% url 'gridsitesync-detail' 'RAL-LCG2' %}?format=json +

+ + + diff --git a/monitoring/urls.py b/monitoring/urls.py index 26e4055..10c97b9 100644 --- a/monitoring/urls.py +++ b/monitoring/urls.py @@ -1,25 +1,12 @@ -"""Monitoring URL Configuration. - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.11/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.conf.urls import url, include - 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) -""" -from django.conf.urls import include, url +from django.urls import include, path from django.contrib import admin +from django.views.generic import TemplateView urlpatterns = [ - url(r'^admin/', admin.site.urls), - url(r'^availability/', include('monitoring.availability.urls')), - url(r'^publishing/', include('monitoring.publishing.urls')), - url(r'^api-auth/', include('rest_framework.urls', - namespace='rest_framework')), + path('admin/', admin.site.urls), + path('', TemplateView.as_view(template_name='home.html'), name='home'), + path('availability/', include('monitoring.availability.urls')), + path('publishing/', include('monitoring.publishing.urls')), + path('benchmarks/', include('monitoring.benchmarks.urls')), + path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), ] diff --git a/monitoring/wsgi.py b/monitoring/wsgi.py index ed72d1b..5fa221b 100644 --- a/monitoring/wsgi.py +++ b/monitoring/wsgi.py @@ -11,6 +11,12 @@ from django.core.wsgi import get_wsgi_application + +# Activate virtualenv +activate_path = os.path.expanduser("/usr/share/DJANGO_MONITORING_APP/venv/bin/activate_this.py") +with open(activate_path) as act: + exec(act.read(), dict(__file__=activate_path)) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitoring.settings") application = get_wsgi_application() diff --git a/requirements.txt b/requirements.txt index 6211931..69f5922 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ -Django==1.11.20 -djangorestframework==3.9.3 -mysqlclient==1.3.4 -pytz==2019.1 +# Pin packages to support and work with py3.6. +Django==3.2.25 +djangorestframework==3.15.1 +pytz==2025.2 +PyMySQL==1.0.2 +pandas==1.1.5 +numpy==1.19.5 # pandas dependency diff --git a/setup.py b/setup.py index 168bf46..f21ce6e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='monitoring', - version='0.1', + version='0.4', packages=find_packages(), scripts=['manage.py'], )