paulmataruso/tacticalrmm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

get jinja templates 100% compatible with reporting

Browse Source
This commit is contained in:
sadnub
2023-05-10 17:36:48 -04:00
parent c645be6b70
commit 9e87318cc5
14 changed files with 309 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.devcontainer/docker-compose.yml
View File

@@ -216,6 +216,7 @@ services:
- "443:4443"
volumes:
- tactical-data-dev:/opt/tactical
- ..:/workspace:cached
volumes:
tactical-data-dev: null

1
.gitignore vendored
View File

@@ -57,3 +57,4 @@ daphne.sock.lock
coverage.xml
setup_dev.yml
11env/
query_schema.json

2
api/tacticalrmm/ee/reporting/management/commands/generate_json_schemas.py
View File

@@ -118,7 +118,7 @@ def generate_schema() -> None:
)
schema = {
"$id": f"https://{djangosettings.ALLOWED_HOSTS[0]}static/reporting/schemas/query_schema.json",
"$id": f"https://{djangosettings.ALLOWED_HOSTS[0]}/static/reporting/schemas/query_schema.json",
"type": "object",
"properties": {
"model": {

6
api/tacticalrmm/ee/reporting/markdown/config.py
View File

@@ -8,8 +8,9 @@ from typing import Optional, Sequence, Union
import markdown
import yaml
from .djangotable_ext import DjangoTableExtension
# from .djangotable_ext import DjangoTableExtension
from .reportasset_ext import ReportAssetExtension
from .ignorejinja_ext import IgnoreJinjaExtension
markdown_ext: "Optional[Sequence[Union[str, markdown.Extension]]]" = [
"ocxsect",
@@ -21,7 +22,8 @@ markdown_ext: "Optional[Sequence[Union[str, markdown.Extension]]]" = [
"full_yaml_metadata",
"attr_list",
ReportAssetExtension(),
DjangoTableExtension(),
IgnoreJinjaExtension(),
# DjangoTableExtension(),
]
extension_config = {

88
api/tacticalrmm/ee/reporting/markdown/djangotable_ext.py
View File

@@ -5,99 +5,15 @@ For details, see: https://license.tacticalrmm.com/ee
"""
import re
from typing import Dict, List, Any
from typing import List, Any
from markdown import Extension, Markdown
from markdown.preprocessors import Preprocessor
from django.apps import apps
from ..constants import REPORTING_MODELS
from ..utils import resolve_model, build_queryset
import pandas as pd
class ResolveModelException(Exception):
pass
def resolve_model(*, data_source: Dict[str, Any]) -> Dict[str, Any]:
tmp_data_source = data_source
# check that model property is present and correct
if "model" in data_source.keys():
for model, app in REPORTING_MODELS:
if data_source["model"].capitalize() == model:
try:
# overwrite model with the model type
tmp_data_source["model"] = apps.get_model(app, model)
return tmp_data_source
except LookupError:
raise ResolveModelException(
f"Model: {model} does not exist in app: {app}"
)
raise ResolveModelException(f"Model lookup failed for {data_source['model']}")
else:
raise ResolveModelException("Model key must be present on data_source")
ALLOWED_OPERATIONS = (
# filtering
"only",
"defer",
"filter",
"exclude",
"limit",
# relations
"select_related",
"prefetch_related",
# operations
"aggregate",
"annotate",
# ordering
"order_by",
)
class InvalidDBOperationException(Exception):
pass
def build_queryset(*, data_source: Dict[str, Any]) -> Any:
local_data_source = data_source
Model = local_data_source.pop("model")
limit = None
columns = local_data_source["only"] if "only" in local_data_source.keys() else None
# create a base reporting queryset
queryset = Model.objects.using("reporting")
for operation, values in local_data_source.items():
if operation not in ALLOWED_OPERATIONS:
raise InvalidDBOperationException(
f"DB operation: {operation} not allowed. Supported operations: only, defer, filter, exclude, limit, select_related, prefetch_related, annotate, aggregate, order_by"
)
if operation == "meta":
continue
elif operation == "limit":
limit = values
elif isinstance(values, list):
queryset = getattr(queryset, operation)(*values)
elif isinstance(values, dict):
queryset = getattr(queryset, operation)(**values)
else:
queryset = getattr(queryset, operation)(values)
if limit:
queryset = queryset[:limit]
if columns:
queryset = queryset.values(*columns)
else:
queryset = queryset.values()
return queryset
RE = re.compile(r"\\table\((.*)\)")

65
api/tacticalrmm/ee/reporting/markdown/ignorejinja_ext.py Normal file
View File

@@ -0,0 +1,65 @@
"""
Copyright (c) 2023-present Amidaware Inc.
This file is subject to the EE License Agreement.
For details, see: https://license.tacticalrmm.com/ee
"""
import re
from typing import List, Any
from markdown import Extension, Markdown
from markdown.preprocessors import Preprocessor
from markdown.postprocessors import Postprocessor
class IgnoreJinjaExtension(Extension):
"""Extension for looking up {% block tag %}"""
def extendMarkdown(self, md: Markdown) -> None:
"""Add IgnoreJinjaExtension to Markdown instance."""
md.preprocessors.register(IgnoreJinjaPreprocessor(md), "preignorejinja", 0)
md.postprocessors.register(IgnoreJinjaPostprocessor(md), "postignorejinja", 0)
PRE_RE = re.compile(r"(\{\%.*\%\})")
class IgnoreJinjaPreprocessor(Preprocessor):
"""
Looks for {% block tag %} and wraps it in an html comment <!--- -->
"""
def run(self, lines: List[str]) -> List[str]:
new_lines: List[str] = []
for line in lines:
m = PRE_RE.search(line)
if m:
tag = m.group(1)
new_line = line.replace(tag, f"<!--- {tag} -->")
new_lines.append(new_line)
else:
new_lines.append(line)
return new_lines
POST_RE = re.compile(r"\<\!\-\-\-\s{1}(\{\%.*\%\})\s{1}\-\-\>")
class IgnoreJinjaPostprocessor(Postprocessor):
"""
Looks for <!-- {{% block tag %}} --> and removes the comment
"""
def run(self, text: str) -> str:
new_lines: List[str] = []
lines = text.split("\n")
for line in lines:
m = POST_RE.search(line)
if m:
tag = m.group(1)
new_line = line.replace(f"<!--- {tag} -->", tag)
new_lines.append(new_line)
else:
new_lines.append(line)
return "\n".join(new_lines)
def makeExtension(*args: Any, **kwargs: Any) -> IgnoreJinjaExtension:
"""set up extension."""
return IgnoreJinjaExtension(*args, **kwargs)

24
api/tacticalrmm/ee/reporting/migrations/0009_reporttemplate_template_variables_and_more.py Normal file
View File

@@ -0,0 +1,24 @@
# Generated by Django 4.1.8 on 2023-05-09 15:44
from django.db import migrations, models
import ee.reporting.storage
class Migration(migrations.Migration):
dependencies = [
('reporting', '0008_reporttemplate_type_alter_reportasset_file'),
]
operations = [
migrations.AddField(
model_name='reporttemplate',
name='template_variables',
field=models.TextField(blank=True, null=True),
),
migrations.AlterField(
model_name='reportdataquery',
name='json_query',
field=models.JSONField(),
),
]

1
api/tacticalrmm/ee/reporting/models.py
View File

@@ -30,6 +30,7 @@ class ReportTemplate(models.Model):
choices=ReportFormatType.choices,
default=ReportFormatType.MARKDOWN,
)
template_variables = models.TextField(null=True, blank=True)
def __str__(self) -> str:
return self.name

2
api/tacticalrmm/ee/reporting/settings.py
View File

@@ -24,7 +24,7 @@ class Settings:
return getattr(
self.settings,
"REPORTING_BASE_URL",
f"https://{djangosettings.ALLOWED_HOSTS[0]}/assets",
f"https://{djangosettings.ALLOWED_HOSTS[0]}",
)

2
api/tacticalrmm/ee/reporting/storage.py
View File

@@ -65,5 +65,5 @@ class ReportAssetStorage(FileSystemStorage):
report_assets_fs = ReportAssetStorage(
location=settings.REPORTING_ASSETS_BASE_PATH,
base_url=f"{settings.REPORTING_BASE_URL}/reporting/assets/files",
base_url=f"{settings.REPORTING_BASE_URL}/reporting/assets/",
)

205
api/tacticalrmm/ee/reporting/utils.py
View File

@@ -4,29 +4,47 @@ This file is subject to the EE License Agreement.
For details, see: https://license.tacticalrmm.com/ee
"""
from rest_framework import status
from rest_framework.response import Response
import yaml
import re
from django.apps import apps
from django.db.models import Model
from weasyprint import HTML, CSS
from weasyprint.text.fonts import FontConfiguration
from jinja2 import Template
from jinja2 import Environment, FunctionLoader
from typing import Dict, Any
from .markdown.config import Markdown
from typing import Optional
from .models import ReportHTMLTemplate, ReportTemplate
from .constants import REPORTING_MODELS
from tacticalrmm.utils import replace_db_values
default_template = """
<html>
<head>
<style>
{{ css }}
</style>
</head>
# this will lookup the Jinja parent template in the DB
# Example: {% extends "MASTER_TEMPLATE_NAME or REPORT_TEMPLATE_NAME" %}
def db_template_loader(template_name):
# trys the ReportHTMLTemplate table and ReportTemplate table
try:
return ReportHTMLTemplate.objects.get(name=template_name).html
except ReportHTMLTemplate.DoesNotExist:
pass
<body>
{{ body }}
</body>
</html>
"""
try:
template = ReportTemplate.objects.get(name=template_name)
return template.template_html if template.type == "html" else template.template_md
except ReportHTMLTemplate.DoesNotExist:
pass
return None
# sets up Jinja environment wiht the db loader template
# comment tags needed to be editted because they conflicted with css properties
env = Environment(
loader=FunctionLoader(db_template_loader),
comment_start_string='{=',
comment_end_string='=}',
)
def generate_pdf(*, html: str, css: str = "") -> bytes:
font_config = FontConfiguration()
@@ -43,16 +61,153 @@ def generate_html(
template: str,
template_type: str,
css: str = "",
html_template: Optional[str] = None
html_template: int = None,
variables: Dict[str,Any] = {},
instance: Model = None
) -> str:
tm = Template(html_template if html_template else default_template)
html_report = tm.render(
css=css,
body=Markdown.convert(template) if template_type == "markdown" else template,
)
return html_report
# convert template from markdown to html if type is markdown
template_string = Markdown.convert(template) if template_type == "markdown" else template
# variables are stored in Markdown.Meta for markdown templates and template_variables field on model for
# html templates.
variables = None
if template_type == "html" and variables:
variables = yaml.safe_load(variables)
elif template_type == "markdown":
variables = Markdown.Meta
# check for variables that need to be replaced with the database values ({{client.name}}, {{agent.hostname}}, etc)
pattern = re.compile("\\{\\{([\\w\\s]+\\.[\\w\\s]+)\\}\\}")
if variables and isinstance(variables, dict):
for key, variable in variables.items():
if isinstance(variable, str):
for string in re.findall(pattern, variable):
value = replace_db_values(string=string, instance=instance, quotes=False)
variable[key] = re.sub("\\{\\{" + string + "\\}\\}", str(value), variable)
# append extends if html master template is configured
if html_template:
try:
html_template_name = ReportHTMLTemplate.objects.get(pk=html_template).name
template_string = f"""{{% extends "{html_template_name}" %}}\n{template_string}"""
except ReportHTMLTemplate.DoesNotExist:
pass
# replace the data_sources with the actual data from DB. This will be passed to the template
# in the form of {{data_sources.data_source_name}}
if isinstance(variables, dict) and "data_sources" in variables and isinstance(variables["data_sources"], dict):
for key, value in variables["data_sources"].items():
data_source = {}
# data_source is referencing a saved data query
if isinstance(value, str):
ReportDataQuery = apps.get_model("reporting", "ReportDataQuery")
try:
data_source = ReportDataQuery.objects.get(
name=value
).json_query
except ReportDataQuery.DoesNotExist:
continue
# inline data source
elif isinstance(value, dict):
data_source = value
_ = data_source.pop("meta") if "meta" in data_source.keys() else None
modified_datasource = resolve_model(data_source=data_source)
queryset = build_queryset(data_source=modified_datasource)
variables["data_sources"][key] = queryset
tm = env.from_string(template_string)
if variables:
return tm.render(css=css, **variables)
else:
return tm.render(css=css)
def notify_error(msg: str) -> Response:
return Response(msg, status=status.HTTP_400_BAD_REQUEST)
class ResolveModelException(Exception):
pass
def resolve_model(*, data_source: Dict[str, Any]) -> Dict[str, Any]:
tmp_data_source = data_source
# check that model property is present and correct
if "model" in data_source.keys():
for model, app in REPORTING_MODELS:
if data_source["model"].capitalize() == model:
try:
# overwrite model with the model type
tmp_data_source["model"] = apps.get_model(app, model)
return tmp_data_source
except LookupError:
raise ResolveModelException(
f"Model: {model} does not exist in app: {app}"
)
raise ResolveModelException(f"Model lookup failed for {data_source['model']}")
else:
raise ResolveModelException("Model key must be present on data_source")
ALLOWED_OPERATIONS = (
# filtering
"only",
"defer",
"filter",
"exclude",
"limit",
# relations
"select_related",
"prefetch_related",
# operations
"aggregate",
"annotate",
# ordering
"order_by",
)
class InvalidDBOperationException(Exception):
pass
def build_queryset(*, data_source: Dict[str, Any]) -> Any:
local_data_source = data_source
Model = local_data_source.pop("model")
limit = None
columns = local_data_source["only"] if "only" in local_data_source.keys() else None
# create a base reporting queryset
queryset = Model.objects.using("reporting")
for operation, values in local_data_source.items():
if operation not in ALLOWED_OPERATIONS:
raise InvalidDBOperationException(
f"DB operation: {operation} not allowed. Supported operations: only, defer, filter, exclude, limit, select_related, prefetch_related, annotate, aggregate, order_by"
)
if operation == "meta":
continue
elif operation == "limit":
limit = values
elif isinstance(values, list):
queryset = getattr(queryset, operation)(*values)
elif isinstance(values, dict):
queryset = getattr(queryset, operation)(**values)
else:
queryset = getattr(queryset, operation)(values)
if limit:
queryset = queryset[:limit]
if columns:
queryset = queryset.values(*columns)
else:
queryset = queryset.values()
return queryset

31
api/tacticalrmm/ee/reporting/views.py
View File

@@ -29,8 +29,8 @@ import shutil
from .storage import report_assets_fs
from .models import ReportTemplate, ReportAsset, ReportHTMLTemplate, ReportDataQuery
from .utils import generate_html, generate_pdf, notify_error
from .utils import generate_html, generate_pdf
from tacticalrmm.utils import notify_error
def path_exists(value: str) -> None:
if not report_assets_fs.exists(value):
@@ -118,26 +118,15 @@ class GenerateReportPreview(APIView):
else None
)
response_format = request.data["format"]
html_report = generate_html(
template=request.data["template_md"] if template_type == "markdown" else request.data["template_html"],
template_type=request.data["type"],
css=template_css,
html_template=template_html,
variables=request.data["template_variables"],
)
# lookup html template
if template_html:
try:
template_html = ReportHTMLTemplate.objects.get(pk=template_html).html
html_report = generate_html(
template=template_md,
template_type=template_type,
css=template_css,
html_template=template_html,
)
except ReportHTMLTemplate.DoesNotExist:
html_report = generate_html(
template=template_md, template_type=template_type, css=template_css
)
else:
html_report = generate_html(
template=template_md, template_type=template_type, css=template_css
)
response_format = request.data["format"]
if response_format == "html":
return Response(html_report)

19
docker/containers/tactical-nginx/entrypoint.sh
View File

@@ -54,6 +54,13 @@ if [[ $DEV -eq 1 ]]; then
proxy_set_header X-Forwarded-Host \$host;
proxy_set_header X-Forwarded-Port \$server_port;
"
STATIC_ASSETS="
location /static/ {
root /workspace/api/tacticalrmm;
add_header "Access-Control-Allow-Origin" "https://${APP_HOST}";
}
"
else
API_NGINX="
#Using variable to disable start checks
@@ -62,6 +69,13 @@ else
include uwsgi_params;
uwsgi_pass \$api;
"
STATIC_ASSETS="
location /static/ {
root ${TACTICAL_DIR}/api/;
add_header "Access-Control-Allow-Origin" "https://${APP_HOST}";
}
"
fi
nginx_config="$(cat << EOF
@@ -75,10 +89,7 @@ server {
${API_NGINX}
}
location /static/ {
root ${TACTICAL_DIR}/api;
add_header "Access-Control-Allow-Origin" "https://${APP_HOST}";
}
${STATIC_ASSETS}
location /private/ {
internal;

2
docker/image-build.sh
View File

@@ -4,7 +4,7 @@ set -o errexit
set -o pipefail
# tactical tactical-frontend tactical-nats tactical-nginx tactical-meshcentral
DOCKER_IMAGES="tactical tactical-frontend tactical-nats tactical-nginx tactical-meshcentral"
DOCKER_IMAGES="tactical-nginx"
cd ..