get jinja templates 100% compatible with reporting
This commit is contained in:
@@ -216,6 +216,7 @@ services:
|
||||
- "443:4443"
|
||||
volumes:
|
||||
- tactical-data-dev:/opt/tactical
|
||||
- ..:/workspace:cached
|
||||
|
||||
volumes:
|
||||
tactical-data-dev: null
|
||||
|
1
.gitignore
vendored
1
.gitignore
vendored
@@ -57,3 +57,4 @@ daphne.sock.lock
|
||||
coverage.xml
|
||||
setup_dev.yml
|
||||
11env/
|
||||
query_schema.json
|
@@ -118,7 +118,7 @@ def generate_schema() -> None:
|
||||
)
|
||||
|
||||
schema = {
|
||||
"$id": f"https://{djangosettings.ALLOWED_HOSTS[0]}static/reporting/schemas/query_schema.json",
|
||||
"$id": f"https://{djangosettings.ALLOWED_HOSTS[0]}/static/reporting/schemas/query_schema.json",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
|
@@ -8,8 +8,9 @@ from typing import Optional, Sequence, Union
|
||||
import markdown
|
||||
import yaml
|
||||
|
||||
from .djangotable_ext import DjangoTableExtension
|
||||
# from .djangotable_ext import DjangoTableExtension
|
||||
from .reportasset_ext import ReportAssetExtension
|
||||
from .ignorejinja_ext import IgnoreJinjaExtension
|
||||
|
||||
markdown_ext: "Optional[Sequence[Union[str, markdown.Extension]]]" = [
|
||||
"ocxsect",
|
||||
@@ -21,7 +22,8 @@ markdown_ext: "Optional[Sequence[Union[str, markdown.Extension]]]" = [
|
||||
"full_yaml_metadata",
|
||||
"attr_list",
|
||||
ReportAssetExtension(),
|
||||
DjangoTableExtension(),
|
||||
IgnoreJinjaExtension(),
|
||||
# DjangoTableExtension(),
|
||||
]
|
||||
|
||||
extension_config = {
|
||||
|
@@ -5,99 +5,15 @@ For details, see: https://license.tacticalrmm.com/ee
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Any
|
||||
from typing import List, Any
|
||||
from markdown import Extension, Markdown
|
||||
from markdown.preprocessors import Preprocessor
|
||||
|
||||
from django.apps import apps
|
||||
from ..constants import REPORTING_MODELS
|
||||
|
||||
from ..utils import resolve_model, build_queryset
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class ResolveModelException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def resolve_model(*, data_source: Dict[str, Any]) -> Dict[str, Any]:
|
||||
tmp_data_source = data_source
|
||||
|
||||
# check that model property is present and correct
|
||||
if "model" in data_source.keys():
|
||||
for model, app in REPORTING_MODELS:
|
||||
if data_source["model"].capitalize() == model:
|
||||
try:
|
||||
# overwrite model with the model type
|
||||
tmp_data_source["model"] = apps.get_model(app, model)
|
||||
return tmp_data_source
|
||||
except LookupError:
|
||||
raise ResolveModelException(
|
||||
f"Model: {model} does not exist in app: {app}"
|
||||
)
|
||||
raise ResolveModelException(f"Model lookup failed for {data_source['model']}")
|
||||
else:
|
||||
raise ResolveModelException("Model key must be present on data_source")
|
||||
|
||||
|
||||
ALLOWED_OPERATIONS = (
|
||||
# filtering
|
||||
"only",
|
||||
"defer",
|
||||
"filter",
|
||||
"exclude",
|
||||
"limit",
|
||||
# relations
|
||||
"select_related",
|
||||
"prefetch_related",
|
||||
# operations
|
||||
"aggregate",
|
||||
"annotate",
|
||||
# ordering
|
||||
"order_by",
|
||||
)
|
||||
|
||||
|
||||
class InvalidDBOperationException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def build_queryset(*, data_source: Dict[str, Any]) -> Any:
|
||||
local_data_source = data_source
|
||||
Model = local_data_source.pop("model")
|
||||
limit = None
|
||||
columns = local_data_source["only"] if "only" in local_data_source.keys() else None
|
||||
|
||||
# create a base reporting queryset
|
||||
queryset = Model.objects.using("reporting")
|
||||
|
||||
for operation, values in local_data_source.items():
|
||||
if operation not in ALLOWED_OPERATIONS:
|
||||
raise InvalidDBOperationException(
|
||||
f"DB operation: {operation} not allowed. Supported operations: only, defer, filter, exclude, limit, select_related, prefetch_related, annotate, aggregate, order_by"
|
||||
)
|
||||
|
||||
if operation == "meta":
|
||||
continue
|
||||
elif operation == "limit":
|
||||
limit = values
|
||||
elif isinstance(values, list):
|
||||
queryset = getattr(queryset, operation)(*values)
|
||||
elif isinstance(values, dict):
|
||||
queryset = getattr(queryset, operation)(**values)
|
||||
else:
|
||||
queryset = getattr(queryset, operation)(values)
|
||||
|
||||
if limit:
|
||||
queryset = queryset[:limit]
|
||||
|
||||
if columns:
|
||||
queryset = queryset.values(*columns)
|
||||
else:
|
||||
queryset = queryset.values()
|
||||
|
||||
return queryset
|
||||
|
||||
|
||||
RE = re.compile(r"\\table\((.*)\)")
|
||||
|
||||
|
||||
|
65
api/tacticalrmm/ee/reporting/markdown/ignorejinja_ext.py
Normal file
65
api/tacticalrmm/ee/reporting/markdown/ignorejinja_ext.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
Copyright (c) 2023-present Amidaware Inc.
|
||||
This file is subject to the EE License Agreement.
|
||||
For details, see: https://license.tacticalrmm.com/ee
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import List, Any
|
||||
from markdown import Extension, Markdown
|
||||
from markdown.preprocessors import Preprocessor
|
||||
from markdown.postprocessors import Postprocessor
|
||||
|
||||
|
||||
class IgnoreJinjaExtension(Extension):
|
||||
"""Extension for looking up {% block tag %}"""
|
||||
|
||||
def extendMarkdown(self, md: Markdown) -> None:
|
||||
"""Add IgnoreJinjaExtension to Markdown instance."""
|
||||
md.preprocessors.register(IgnoreJinjaPreprocessor(md), "preignorejinja", 0)
|
||||
md.postprocessors.register(IgnoreJinjaPostprocessor(md), "postignorejinja", 0)
|
||||
|
||||
PRE_RE = re.compile(r"(\{\%.*\%\})")
|
||||
|
||||
class IgnoreJinjaPreprocessor(Preprocessor):
|
||||
"""
|
||||
Looks for {% block tag %} and wraps it in an html comment <!--- -->
|
||||
"""
|
||||
|
||||
def run(self, lines: List[str]) -> List[str]:
|
||||
new_lines: List[str] = []
|
||||
for line in lines:
|
||||
m = PRE_RE.search(line)
|
||||
if m:
|
||||
tag = m.group(1)
|
||||
new_line = line.replace(tag, f"<!--- {tag} -->")
|
||||
new_lines.append(new_line)
|
||||
else:
|
||||
new_lines.append(line)
|
||||
|
||||
return new_lines
|
||||
|
||||
POST_RE = re.compile(r"\<\!\-\-\-\s{1}(\{\%.*\%\})\s{1}\-\-\>")
|
||||
|
||||
class IgnoreJinjaPostprocessor(Postprocessor):
|
||||
"""
|
||||
Looks for <!-- {{% block tag %}} --> and removes the comment
|
||||
"""
|
||||
|
||||
def run(self, text: str) -> str:
|
||||
new_lines: List[str] = []
|
||||
lines = text.split("\n")
|
||||
for line in lines:
|
||||
m = POST_RE.search(line)
|
||||
if m:
|
||||
tag = m.group(1)
|
||||
new_line = line.replace(f"<!--- {tag} -->", tag)
|
||||
new_lines.append(new_line)
|
||||
else:
|
||||
new_lines.append(line)
|
||||
return "\n".join(new_lines)
|
||||
|
||||
|
||||
def makeExtension(*args: Any, **kwargs: Any) -> IgnoreJinjaExtension:
|
||||
"""set up extension."""
|
||||
return IgnoreJinjaExtension(*args, **kwargs)
|
@@ -0,0 +1,24 @@
|
||||
# Generated by Django 4.1.8 on 2023-05-09 15:44
|
||||
|
||||
from django.db import migrations, models
|
||||
import ee.reporting.storage
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('reporting', '0008_reporttemplate_type_alter_reportasset_file'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='reporttemplate',
|
||||
name='template_variables',
|
||||
field=models.TextField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='reportdataquery',
|
||||
name='json_query',
|
||||
field=models.JSONField(),
|
||||
),
|
||||
]
|
@@ -30,7 +30,8 @@ class ReportTemplate(models.Model):
|
||||
choices=ReportFormatType.choices,
|
||||
default=ReportFormatType.MARKDOWN,
|
||||
)
|
||||
|
||||
template_variables = models.TextField(null=True, blank=True)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
@@ -24,7 +24,7 @@ class Settings:
|
||||
return getattr(
|
||||
self.settings,
|
||||
"REPORTING_BASE_URL",
|
||||
f"https://{djangosettings.ALLOWED_HOSTS[0]}/assets",
|
||||
f"https://{djangosettings.ALLOWED_HOSTS[0]}",
|
||||
)
|
||||
|
||||
|
||||
|
@@ -65,5 +65,5 @@ class ReportAssetStorage(FileSystemStorage):
|
||||
|
||||
report_assets_fs = ReportAssetStorage(
|
||||
location=settings.REPORTING_ASSETS_BASE_PATH,
|
||||
base_url=f"{settings.REPORTING_BASE_URL}/reporting/assets/files",
|
||||
base_url=f"{settings.REPORTING_BASE_URL}/reporting/assets/",
|
||||
)
|
||||
|
@@ -4,29 +4,47 @@ This file is subject to the EE License Agreement.
|
||||
For details, see: https://license.tacticalrmm.com/ee
|
||||
"""
|
||||
|
||||
from rest_framework import status
|
||||
from rest_framework.response import Response
|
||||
import yaml
|
||||
import re
|
||||
|
||||
from django.apps import apps
|
||||
from django.db.models import Model
|
||||
from weasyprint import HTML, CSS
|
||||
from weasyprint.text.fonts import FontConfiguration
|
||||
from jinja2 import Template
|
||||
from jinja2 import Environment, FunctionLoader
|
||||
from typing import Dict, Any
|
||||
|
||||
from .markdown.config import Markdown
|
||||
from typing import Optional
|
||||
from .models import ReportHTMLTemplate, ReportTemplate
|
||||
from .constants import REPORTING_MODELS
|
||||
|
||||
from tacticalrmm.utils import replace_db_values
|
||||
|
||||
|
||||
default_template = """
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
{{ css }}
|
||||
</style>
|
||||
</head>
|
||||
# this will lookup the Jinja parent template in the DB
|
||||
# Example: {% extends "MASTER_TEMPLATE_NAME or REPORT_TEMPLATE_NAME" %}
|
||||
def db_template_loader(template_name):
|
||||
# trys the ReportHTMLTemplate table and ReportTemplate table
|
||||
try:
|
||||
return ReportHTMLTemplate.objects.get(name=template_name).html
|
||||
except ReportHTMLTemplate.DoesNotExist:
|
||||
pass
|
||||
|
||||
<body>
|
||||
{{ body }}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
try:
|
||||
template = ReportTemplate.objects.get(name=template_name)
|
||||
return template.template_html if template.type == "html" else template.template_md
|
||||
except ReportHTMLTemplate.DoesNotExist:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
# sets up Jinja environment wiht the db loader template
|
||||
# comment tags needed to be editted because they conflicted with css properties
|
||||
env = Environment(
|
||||
loader=FunctionLoader(db_template_loader),
|
||||
comment_start_string='{=',
|
||||
comment_end_string='=}',
|
||||
)
|
||||
|
||||
def generate_pdf(*, html: str, css: str = "") -> bytes:
|
||||
font_config = FontConfiguration()
|
||||
@@ -43,16 +61,153 @@ def generate_html(
|
||||
template: str,
|
||||
template_type: str,
|
||||
css: str = "",
|
||||
html_template: Optional[str] = None
|
||||
html_template: int = None,
|
||||
variables: Dict[str,Any] = {},
|
||||
instance: Model = None
|
||||
) -> str:
|
||||
tm = Template(html_template if html_template else default_template)
|
||||
html_report = tm.render(
|
||||
css=css,
|
||||
body=Markdown.convert(template) if template_type == "markdown" else template,
|
||||
)
|
||||
|
||||
# convert template from markdown to html if type is markdown
|
||||
template_string = Markdown.convert(template) if template_type == "markdown" else template
|
||||
|
||||
return html_report
|
||||
# variables are stored in Markdown.Meta for markdown templates and template_variables field on model for
|
||||
# html templates.
|
||||
variables = None
|
||||
if template_type == "html" and variables:
|
||||
variables = yaml.safe_load(variables)
|
||||
elif template_type == "markdown":
|
||||
variables = Markdown.Meta
|
||||
|
||||
# check for variables that need to be replaced with the database values ({{client.name}}, {{agent.hostname}}, etc)
|
||||
pattern = re.compile("\\{\\{([\\w\\s]+\\.[\\w\\s]+)\\}\\}")
|
||||
|
||||
if variables and isinstance(variables, dict):
|
||||
for key, variable in variables.items():
|
||||
if isinstance(variable, str):
|
||||
for string in re.findall(pattern, variable):
|
||||
value = replace_db_values(string=string, instance=instance, quotes=False)
|
||||
|
||||
variable[key] = re.sub("\\{\\{" + string + "\\}\\}", str(value), variable)
|
||||
|
||||
# append extends if html master template is configured
|
||||
if html_template:
|
||||
try:
|
||||
html_template_name = ReportHTMLTemplate.objects.get(pk=html_template).name
|
||||
|
||||
template_string = f"""{{% extends "{html_template_name}" %}}\n{template_string}"""
|
||||
except ReportHTMLTemplate.DoesNotExist:
|
||||
pass
|
||||
|
||||
# replace the data_sources with the actual data from DB. This will be passed to the template
|
||||
# in the form of {{data_sources.data_source_name}}
|
||||
if isinstance(variables, dict) and "data_sources" in variables and isinstance(variables["data_sources"], dict):
|
||||
for key, value in variables["data_sources"].items():
|
||||
|
||||
data_source = {}
|
||||
# data_source is referencing a saved data query
|
||||
if isinstance(value, str):
|
||||
ReportDataQuery = apps.get_model("reporting", "ReportDataQuery")
|
||||
try:
|
||||
data_source = ReportDataQuery.objects.get(
|
||||
name=value
|
||||
).json_query
|
||||
except ReportDataQuery.DoesNotExist:
|
||||
continue
|
||||
|
||||
# inline data source
|
||||
elif isinstance(value, dict):
|
||||
data_source = value
|
||||
|
||||
_ = data_source.pop("meta") if "meta" in data_source.keys() else None
|
||||
|
||||
modified_datasource = resolve_model(data_source=data_source)
|
||||
queryset = build_queryset(data_source=modified_datasource)
|
||||
variables["data_sources"][key] = queryset
|
||||
|
||||
tm = env.from_string(template_string)
|
||||
if variables:
|
||||
return tm.render(css=css, **variables)
|
||||
else:
|
||||
return tm.render(css=css)
|
||||
|
||||
|
||||
def notify_error(msg: str) -> Response:
|
||||
return Response(msg, status=status.HTTP_400_BAD_REQUEST)
|
||||
class ResolveModelException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def resolve_model(*, data_source: Dict[str, Any]) -> Dict[str, Any]:
|
||||
tmp_data_source = data_source
|
||||
|
||||
# check that model property is present and correct
|
||||
if "model" in data_source.keys():
|
||||
for model, app in REPORTING_MODELS:
|
||||
if data_source["model"].capitalize() == model:
|
||||
try:
|
||||
# overwrite model with the model type
|
||||
tmp_data_source["model"] = apps.get_model(app, model)
|
||||
return tmp_data_source
|
||||
except LookupError:
|
||||
raise ResolveModelException(
|
||||
f"Model: {model} does not exist in app: {app}"
|
||||
)
|
||||
raise ResolveModelException(f"Model lookup failed for {data_source['model']}")
|
||||
else:
|
||||
raise ResolveModelException("Model key must be present on data_source")
|
||||
|
||||
|
||||
ALLOWED_OPERATIONS = (
|
||||
# filtering
|
||||
"only",
|
||||
"defer",
|
||||
"filter",
|
||||
"exclude",
|
||||
"limit",
|
||||
# relations
|
||||
"select_related",
|
||||
"prefetch_related",
|
||||
# operations
|
||||
"aggregate",
|
||||
"annotate",
|
||||
# ordering
|
||||
"order_by",
|
||||
)
|
||||
|
||||
|
||||
class InvalidDBOperationException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def build_queryset(*, data_source: Dict[str, Any]) -> Any:
|
||||
local_data_source = data_source
|
||||
Model = local_data_source.pop("model")
|
||||
limit = None
|
||||
columns = local_data_source["only"] if "only" in local_data_source.keys() else None
|
||||
|
||||
# create a base reporting queryset
|
||||
queryset = Model.objects.using("reporting")
|
||||
|
||||
for operation, values in local_data_source.items():
|
||||
if operation not in ALLOWED_OPERATIONS:
|
||||
raise InvalidDBOperationException(
|
||||
f"DB operation: {operation} not allowed. Supported operations: only, defer, filter, exclude, limit, select_related, prefetch_related, annotate, aggregate, order_by"
|
||||
)
|
||||
|
||||
if operation == "meta":
|
||||
continue
|
||||
elif operation == "limit":
|
||||
limit = values
|
||||
elif isinstance(values, list):
|
||||
queryset = getattr(queryset, operation)(*values)
|
||||
elif isinstance(values, dict):
|
||||
queryset = getattr(queryset, operation)(**values)
|
||||
else:
|
||||
queryset = getattr(queryset, operation)(values)
|
||||
|
||||
if limit:
|
||||
queryset = queryset[:limit]
|
||||
|
||||
if columns:
|
||||
queryset = queryset.values(*columns)
|
||||
else:
|
||||
queryset = queryset.values()
|
||||
|
||||
return queryset
|
||||
|
@@ -29,8 +29,8 @@ import shutil
|
||||
|
||||
from .storage import report_assets_fs
|
||||
from .models import ReportTemplate, ReportAsset, ReportHTMLTemplate, ReportDataQuery
|
||||
from .utils import generate_html, generate_pdf, notify_error
|
||||
|
||||
from .utils import generate_html, generate_pdf
|
||||
from tacticalrmm.utils import notify_error
|
||||
|
||||
def path_exists(value: str) -> None:
|
||||
if not report_assets_fs.exists(value):
|
||||
@@ -117,28 +117,17 @@ class GenerateReportPreview(APIView):
|
||||
if "template_html" in request.data.keys()
|
||||
else None
|
||||
)
|
||||
|
||||
html_report = generate_html(
|
||||
template=request.data["template_md"] if template_type == "markdown" else request.data["template_html"],
|
||||
template_type=request.data["type"],
|
||||
css=template_css,
|
||||
html_template=template_html,
|
||||
variables=request.data["template_variables"],
|
||||
)
|
||||
|
||||
response_format = request.data["format"]
|
||||
|
||||
# lookup html template
|
||||
if template_html:
|
||||
try:
|
||||
template_html = ReportHTMLTemplate.objects.get(pk=template_html).html
|
||||
html_report = generate_html(
|
||||
template=template_md,
|
||||
template_type=template_type,
|
||||
css=template_css,
|
||||
html_template=template_html,
|
||||
)
|
||||
except ReportHTMLTemplate.DoesNotExist:
|
||||
html_report = generate_html(
|
||||
template=template_md, template_type=template_type, css=template_css
|
||||
)
|
||||
else:
|
||||
html_report = generate_html(
|
||||
template=template_md, template_type=template_type, css=template_css
|
||||
)
|
||||
|
||||
|
||||
if response_format == "html":
|
||||
return Response(html_report)
|
||||
else:
|
||||
|
@@ -54,6 +54,13 @@ if [[ $DEV -eq 1 ]]; then
|
||||
proxy_set_header X-Forwarded-Host \$host;
|
||||
proxy_set_header X-Forwarded-Port \$server_port;
|
||||
"
|
||||
|
||||
STATIC_ASSETS="
|
||||
location /static/ {
|
||||
root /workspace/api/tacticalrmm;
|
||||
add_header "Access-Control-Allow-Origin" "https://${APP_HOST}";
|
||||
}
|
||||
"
|
||||
else
|
||||
API_NGINX="
|
||||
#Using variable to disable start checks
|
||||
@@ -62,6 +69,13 @@ else
|
||||
include uwsgi_params;
|
||||
uwsgi_pass \$api;
|
||||
"
|
||||
|
||||
STATIC_ASSETS="
|
||||
location /static/ {
|
||||
root ${TACTICAL_DIR}/api/;
|
||||
add_header "Access-Control-Allow-Origin" "https://${APP_HOST}";
|
||||
}
|
||||
"
|
||||
fi
|
||||
|
||||
nginx_config="$(cat << EOF
|
||||
@@ -75,10 +89,7 @@ server {
|
||||
${API_NGINX}
|
||||
}
|
||||
|
||||
location /static/ {
|
||||
root ${TACTICAL_DIR}/api;
|
||||
add_header "Access-Control-Allow-Origin" "https://${APP_HOST}";
|
||||
}
|
||||
${STATIC_ASSETS}
|
||||
|
||||
location /private/ {
|
||||
internal;
|
||||
|
@@ -4,7 +4,7 @@ set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
# tactical tactical-frontend tactical-nats tactical-nginx tactical-meshcentral
|
||||
DOCKER_IMAGES="tactical tactical-frontend tactical-nats tactical-nginx tactical-meshcentral"
|
||||
DOCKER_IMAGES="tactical-nginx"
|
||||
|
||||
cd ..
|
||||
|
||||
|
Reference in New Issue
Block a user