paulmataruso/tacticalrmm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more improvements

Browse Source
This commit is contained in:
sadnub
2023-05-26 10:52:17 -04:00
parent 29778ca19e
commit 1c0aa55e7a
13 changed files with 219 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/tacticalrmm/core/views.py
View File

@@ -355,7 +355,7 @@ class RunURLAction(APIView):
from agents.models import Agent
from clients.models import Client, Site
from tacticalrmm.utils import replace_db_values
from tacticalrmm.utils import get_db_value
if "agent_id" in request.data.keys():
if not _has_perm_on_agent(request.user, request.data["agent_id"]):
@@ -382,7 +382,7 @@ class RunURLAction(APIView):
url_pattern = action.pattern
for string in re.findall(pattern, action.pattern):
value = replace_db_values(string=string, instance=instance, quotes=False)
value = get_db_value(string=string, instance=instance, quotes=False)
url_pattern = re.sub("\\{\\{" + string + "\\}\\}", str(value), url_pattern)

4
api/tacticalrmm/ee/reporting/markdown/config.py
View File

@@ -8,8 +8,6 @@ from typing import Optional, Sequence, Union
import markdown
import yaml
# from .djangotable_ext import DjangoTableExtension
from .reportasset_ext import ReportAssetExtension
from .ignorejinja_ext import IgnoreJinjaExtension
markdown_ext: "Optional[Sequence[Union[str, markdown.Extension]]]" = [
@@ -21,9 +19,7 @@ markdown_ext: "Optional[Sequence[Union[str, markdown.Extension]]]" = [
"fenced_code",
"full_yaml_metadata",
"attr_list",
ReportAssetExtension(),
IgnoreJinjaExtension(),
# DjangoTableExtension(),
]
extension_config = {

83
api/tacticalrmm/ee/reporting/markdown/djangotable_ext.py
View File

@@ -1,83 +0,0 @@
"""
Copyright (c) 2023-present Amidaware Inc.
This file is subject to the EE License Agreement.
For details, see: https://license.tacticalrmm.com/ee
"""
import re
from typing import List, Any
from markdown import Extension, Markdown
from markdown.preprocessors import Preprocessor
from django.apps import apps
from ..utils import resolve_model, build_queryset
import pandas as pd
RE = re.compile(r"\\table\((.*)\)")
class DjangoTableExtension(Extension):
"""Extension for parsing Django querysets into markdown tables"""
def extendMarkdown(self, md: Markdown) -> None:
"""Add DjangoTableExtension to Markdown instance."""
md.preprocessors.register(DjangoTablePreprocessor(md), "djangotable", 0)
class DjangoTablePreprocessor(Preprocessor):
"""
Looks for \\table(datasource) and executes the query and uses pandas to convert to markdown table.
Uses a special reporting connection to limit access to SELECTS and only certain tables.
"""
def run(self, lines: List[str]) -> List[str]:
inline_data_sources = (
self.md.Meta["data_sources"]
if hasattr(self.md, "Meta")
and self.md.Meta
and "data_sources" in self.md.Meta.keys()
else None
)
new_lines: List[str] = []
for line in lines:
m = RE.search(line)
if m:
data_query_name = m.group(1)
if (
inline_data_sources
and data_query_name in inline_data_sources.keys()
):
data_source = inline_data_sources[m.group(1)]
else:
# no inline data sources found in yaml, check if it exists in DB
ReportDataQuery = apps.get_model("reporting", "ReportDataQuery")
try:
data_source = ReportDataQuery.objects.get(
name=data_query_name
).json_query
except ReportDataQuery.DoesNotExist:
new_lines.append(line)
continue
meta = data_source.pop("meta") if "meta" in data_source.keys() else None
modified_datasource = resolve_model(data_source=data_source)
queryset = build_queryset(data_source=modified_datasource)
df = pd.DataFrame.from_records(queryset)
if meta:
if "rename_columns" in meta.keys():
df.rename(columns=meta["rename_columns"], inplace=True)
new_lines = new_lines + df.to_markdown(index=False).split("\n")
else:
new_lines.append(line)
return new_lines
def makeExtension(*args: Any, **kwargs: Any) -> DjangoTableExtension:
"""set up extension."""
return DjangoTableExtension(*args, *kwargs)

59
api/tacticalrmm/ee/reporting/markdown/reportasset_ext.py
View File

@@ -1,59 +0,0 @@
"""
Copyright (c) 2023-present Amidaware Inc.
This file is subject to the EE License Agreement.
For details, see: https://license.tacticalrmm.com/ee
"""
import re
from typing import List, Any, TYPE_CHECKING
from markdown import Extension, Markdown
from markdown.preprocessors import Preprocessor
from django.apps import apps
if TYPE_CHECKING:
from ..models import ReportAsset as ReportAssetType
# looking for asset://123e4567-e89b-12d3-a456-426614174000 and replaces with actual path to render image
RE = re.compile(
r"asset://([0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12})"
)
class ReportAssetExtension(Extension):
"""Extension for looking up asset://{uuid4} urls in the DB"""
def extendMarkdown(self, md: Markdown) -> None:
"""Add ReportAssetExtension to Markdown instance."""
md.preprocessors.register(ReportAssetPreprocessor(md), "reportasset", 0)
class ReportAssetPreprocessor(Preprocessor):
"""
Looks for asset://123e4567-e89b-12d3-a456-426614174000 and replaces with actual
path on the file system to render image
"""
def run(self, lines: List[str]) -> List[str]:
new_lines: List[str] = []
for line in lines:
m = RE.search(line)
if m:
asset_id = m.group(1)
ReportAsset = apps.get_model("reporting", "ReportAsset")
asset: "ReportAssetType" = ReportAsset.objects.get(id=asset_id)
new_line = line.replace(f"asset://{asset_id}", f"{asset.file.url}?id={asset_id}")
new_lines.append(new_line)
else:
new_lines.append(line)
return new_lines
def makeExtension(*args: Any, **kwargs: Any) -> ReportAssetExtension:
"""set up extension."""
return ReportAssetExtension(*args, **kwargs)

25
api/tacticalrmm/ee/reporting/migrations/0010_reporttemplate_depends_on_alter_reportasset_file.py Normal file
View File

@@ -0,0 +1,25 @@
# Generated by Django 4.1.8 on 2023-05-13 03:26
import django.contrib.postgres.fields
from django.db import migrations, models
import ee.reporting.storage
class Migration(migrations.Migration):
dependencies = [
('reporting', '0009_reporttemplate_template_variables_and_more'),
]
operations = [
migrations.AddField(
model_name='reporttemplate',
name='depends_on',
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=20), default=list, size=None),
),
migrations.AlterField(
model_name='reportasset',
name='file',
field=models.FileField(storage=ee.reporting.storage.ReportAssetStorage(base_url='https://api.simplermm.com/reporting/assets/', location='/opt/tactical/reporting/assets'), unique=True, upload_to=''),
),
]

4
api/tacticalrmm/ee/reporting/models.py
View File

@@ -5,6 +5,7 @@ For details, see: https://license.tacticalrmm.com/ee
"""
from django.db import models
from django.contrib.postgres.fields import ArrayField
import uuid
from .storage import report_assets_fs
@@ -30,7 +31,8 @@ class ReportTemplate(models.Model):
choices=ReportFormatType.choices,
default=ReportFormatType.MARKDOWN,
)
template_variables = models.TextField(null=True, blank=True)
template_variables = models.TextField(blank=True, default="")
depends_on = ArrayField(models.CharField(max_length=20, blank=True), blank=True, default=list)
def __str__(self) -> str:
return self.name

2
api/tacticalrmm/ee/reporting/urls.py
View File

@@ -11,7 +11,7 @@ urlpatterns = [
# report templates
path("templates/", views.GetAddReportTemplate.as_view()),
path("templates/<int:pk>/", views.GetEditDeleteReportTemplate.as_view()),
path("templates/<int:pk>/run/", views.GenerateSavedReport.as_view()),
path("templates/<int:pk>/run/", views.GenerateReport.as_view()),
path("templates/preview/", views.GenerateReportPreview.as_view()),
# report assets
path("assets/", views.GetReportAssets.as_view()),

77
api/tacticalrmm/ee/reporting/utils.py
View File

@@ -8,17 +8,22 @@ import yaml
import re
from django.apps import apps
from django.db.models import Model
from weasyprint import HTML, CSS
from weasyprint.text.fonts import FontConfiguration
from jinja2 import Environment, FunctionLoader
from typing import Dict, Any
from typing import Dict, Any, Literal
from .markdown.config import Markdown
from .models import ReportHTMLTemplate, ReportTemplate
from .models import ReportHTMLTemplate, ReportTemplate, ReportAsset
from .constants import REPORTING_MODELS
from tacticalrmm.utils import replace_db_values
from tacticalrmm.utils import get_db_value
# regex for db data replacement
# will return 3 groups of matches in a tuple when uses with re.findall
# {{client.name}}, client.name, client
RE_DB_VALUE = re.compile(r'(\{\{\s*((client|site|agent|global)\.{1}[\w\s\d]+)\s*\}\})')
# this will lookup the Jinja parent template in the DB
@@ -62,31 +67,45 @@ def generate_html(
template_type: str,
css: str = "",
html_template: int = None,
variables: Dict[str,Any] = {},
instance: Model = None
variables: str = "",
dependencies: Dict[str, int] = {}
) -> str:
# convert template from markdown to html if type is markdown
template_string = Markdown.convert(template) if template_type == "markdown" else template
# variables are stored in Markdown.Meta for markdown templates and template_variables field on model for
# html templates.
variables = None
if template_type == "html" and variables:
variables = yaml.safe_load(variables)
elif template_type == "markdown":
variables = Markdown.Meta
print(variables)
# load yaml variables if they exist
variables = yaml.safe_load(variables) or {}
# check for variables that need to be replaced with the database values ({{client.name}}, {{agent.hostname}}, etc)
pattern = re.compile("\\{\\{([\\w\\s]+\\.[\\w\\s]+)\\}\\}")
if variables and isinstance(variables, dict):
if variables:
for key, variable in variables.items():
if isinstance(variable, str):
for string in re.findall(pattern, variable):
value = replace_db_values(string=string, instance=instance, quotes=False)
for string, prop, model in re.findall(RE_DB_VALUE, variable):
value = ""
# will be agent, site, client, or global
if model == "global":
value = get_db_value(string=prop)
elif model in ["client", "site", "agent"]:
if model == "client" and "client" in dependencies.keys():
Model = apps.get_model("clients", "Client")
instance = Model.objects.get(id=dependencies["client"])
del dependencies["client"]
elif model == "site" and "site" in dependencies.keys():
Model = apps.get_model("clients", "Site")
instance = Model.objects.get(id=dependencies["site"])
del dependencies["site"]
elif model == "agent" and "agent" in dependencies.keys():
Model = apps.get_model("agents", "Agent")
instance = Model.objects.get(agent_id=dependencies["agent"])
del dependencies["agent"]
else:
instance = None
variable[key] = re.sub("\\{\\{" + string + "\\}\\}", str(value), variable)
value = get_db_value(string=prop, instance=instance) if instance else None
if value:
variables[key] = variable.replace(string, str(value))
# append extends if html master template is configured
if html_template:
@@ -124,6 +143,9 @@ def generate_html(
variables["data_sources"][key] = queryset
tm = env.from_string(template_string)
print(dependencies)
print(variables)
variables = {**variables, **dependencies}
if variables:
return tm.render(css=css, **variables)
else:
@@ -211,3 +233,20 @@ def build_queryset(*, data_source: Dict[str, Any]) -> Any:
queryset = queryset.values()
return queryset
def normalize_asset_url(text: str, type: Literal["pdf", "html"]):
RE_ASSET_URL = re.compile(r"(asset://([0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}))")
new_text = text
for url, id in re.findall(RE_ASSET_URL,text):
try:
asset = ReportAsset.objects.get(id=id)
if type == "html":
new_text = new_text.replace(f"asset://{id}", f"{asset.file.url}?id={id}")
else:
new_text = new_text.replace(f"{url}", f"file://{asset.file.path}")
except ReportAsset.DoesNotExist:
pass
return new_text

83
api/tacticalrmm/ee/reporting/views.py
View File

@@ -27,10 +27,11 @@ from django.shortcuts import get_object_or_404
import os
import shutil
import uuid
from .storage import report_assets_fs
from .models import ReportTemplate, ReportAsset, ReportHTMLTemplate, ReportDataQuery
from .utils import generate_html, generate_pdf
from .utils import generate_html, generate_pdf, normalize_asset_url
from tacticalrmm.utils import notify_error
def path_exists(value: str) -> None:
@@ -49,8 +50,13 @@ class GetAddReportTemplate(APIView):
serializer_class = ReportTemplateSerializer
def get(self, request: Request) -> Response:
reports = ReportTemplate.objects.all()
return Response(ReportTemplateSerializer(reports, many=True).data)
depends_on = request.query_params.getlist("dependsOn[]", [])
if depends_on:
templates = ReportTemplate.objects.filter(depends_on__overlap=depends_on)
else:
templates = ReportTemplate.objects.all()
return Response(ReportTemplateSerializer(templates, many=True).data)
def post(self, request: Request) -> Response:
serializer = ReportTemplateSerializer(data=request.data)
@@ -86,49 +92,58 @@ class GetEditDeleteReportTemplate(APIView):
return Response()
class GenerateSavedReport(APIView):
class GenerateReport(APIView):
def post(self, request: Request, pk: int) -> Union[FileResponse, Response]:
template = get_object_or_404(ReportTemplate, pk=pk)
format = request.data["format"]
html_report = generate_html(
template=template.template_md,
template_type=template.type,
css=template.template_css if template.template_css else "",
html_template=template.template_html.html
html_template=template.template_html.id
if template.template_html
else None,
variables=template.template_variables,
dependencies=request.data["dependencies"]
)
pdf_bytes = generate_pdf(html=html_report)
html_report = normalize_asset_url(html_report, format)
return FileResponse(
ContentFile(pdf_bytes),
content_type="application/pdf",
filename=f"{template.name}.pdf",
)
if format == "html":
return Response(html_report)
elif format == "pdf":
pdf_bytes = generate_pdf(html=html_report)
return FileResponse(
ContentFile(pdf_bytes),
content_type="application/pdf",
filename=f"{template.name}.pdf",
)
else:
notify_error("Report format is incorrect.")
class GenerateReportPreview(APIView):
def post(self, request: Request) -> Union[FileResponse, Response]:
template_md = request.data["template_md"]
template_css = request.data["template_css"]
template_type = request.data["type"]
template_html = (
request.data["template_html"]
if "template_html" in request.data.keys()
else None
)
html_report = generate_html(
template=template_md,
template_type=template_type,
css=template_css,
html_template=template_html,
template=request.data["template_md"],
template_type=request.data["type"],
css=request.data["template_css"],
html_template=(
request.data["template_html"]
if "template_html" in request.data.keys()
else None
),
variables=request.data["template_variables"],
dependencies=request.data["dependencies"]
)
response_format = request.data["format"]
html_report = normalize_asset_url(html_report, response_format)
if response_format == "html":
return Response(html_report)
else:
@@ -506,11 +521,15 @@ class NginxRedirect(APIView):
def get(self, request: Request, path: str) -> HttpResponse:
id = request.query_params.get("id", "")
asset = ReportAsset.objects.get(id=id)
new_path = path.split("?")[0]
if asset.file.name == new_path:
response = HttpResponse(status=200)
response["X-Accel-Redirect"] = "/assets/" + new_path
return response
else:
raise PermissionDenied()
try:
asset_uuid = uuid.UUID(id, version=4)
asset = get_object_or_404(ReportAsset, id=asset_uuid)
new_path = path.split("?")[0]
if asset.file.name == new_path:
response = HttpResponse(status=200)
response["X-Accel-Redirect"] = "/assets/" + new_path
return response
else:
raise PermissionDenied()
except ValueError:
notify_error("There was a error processing the request")

4
api/tacticalrmm/requirements.txt
View File

@@ -40,9 +40,9 @@ websockets==11.0.3
zipp==3.17.0
meshctrl==0.1.15
pandas==1.4.0
kaleido==0.2.1.post1
jinja2==3.0.3
markdown==3.3.6
markdown-full-yaml-metadata==2.1.0
tabulate==0.8.9
plotly==5.14.1
weasyprint==54.3
ocxsect==0.1.5

4
api/tacticalrmm/scripts/models.py
View File

@@ -9,7 +9,7 @@ from django.db.models.fields import CharField, TextField
from logs.models import BaseAuditModel
from tacticalrmm.constants import ScriptShell, ScriptType
from tacticalrmm.utils import replace_db_values
from tacticalrmm.utils import replace_arg_db_values
class Script(BaseAuditModel):
@@ -208,7 +208,7 @@ class Script(BaseAuditModel):
if match := pattern.match(arg):
# only get the match between the () in regex
string = match.group(1)
value = replace_db_values(
value = replace_arg_db_values(
string=string,
instance=agent,
shell=shell,

173
api/tacticalrmm/tacticalrmm/utils.py
View File

@@ -287,131 +287,98 @@ def get_latest_trmm_ver() -> str:
return "error"
def replace_db_values(
string: str, instance=None, shell: str = None, quotes=True # type:ignore
) -> Union[str, None]:
from clients.models import Client, Site
# Receives something like {{ client.name }} and a Model instance of Client, Site, or Agent. If an
# agent instance is passed it will resolve the value of agent.client.name and return the agent's client name.
#
# You can query custom fields by using their name. {{ site.Custom Field Name }}
#
# This will recursively lookup values for relations. {{ client.site.id }}
#
# You can also use {{ global.value }} without an obj instance to use the global key store
def get_db_value(*, string: str, instance=None) -> Union[str, List, True, False, None]:
from core.models import CustomField, GlobalKVStore
# split by period if exists. First should be model and second should be property i.e {{client.name}}
temp = string.split(".")
# check for model and property
if len(temp) < 2:
# ignore arg since it is invalid
return ""
# get properties into an array
props = string.strip().split(".")
# value is in the global keystore and replace value
if temp[0] == "global":
if GlobalKVStore.objects.filter(name=temp[1]).exists():
value = GlobalKVStore.objects.get(name=temp[1]).value
return f"'{value}'" if quotes else value
else:
if props[0] == "global" and len(props) == 2:
try:
return GlobalKVStore.objects.get(name=props[1]).value
except GlobalKVStore.DoesNotExist:
DebugLog.error(
log_type=DebugLogType.SCRIPTING,
message=f"Couldn't lookup value for: {string}. Make sure it exists in CoreSettings > Key Store", # type:ignore
)
return ""
return None
if not instance:
# instance must be set if not global property
return ""
return None
if temp[0] == "client":
model = "client"
if isinstance(instance, Client):
obj = instance
elif hasattr(instance, "client"):
obj = instance.client
instance_value = instance
iterations = 0
# look through all properties and return the value
for prop in props:
iterations += 1
if hasattr(instance_value, prop):
value = getattr(instance_value, prop)
if callable(value):
return
instance_value = getattr(instance_value, prop)
elif iterations == 2:
try:
field = CustomField.objects.get(model=props[0], name=prop)
model_fields = getattr(field, f"{props[0]}_fields")
try:
value = model_fields.get(**{props[0]: instance}).value
return value if field.type != CustomFieldType.CHECKBOX else bool(field.default_value)
except:
return field.default_value if field.type != CustomFieldType.CHECKBOX else bool(field.default_value)
except CustomField.DoesNotExist:
return None
elif iterations == 1:
# if the first property i.e: client is the instance then we can skip trying to lookup properties
pass
else:
obj = None
elif temp[0] == "site":
model = "site"
if isinstance(instance, Site):
obj = instance
elif hasattr(instance, "site"):
obj = instance.site
else:
obj = None
elif temp[0] == "agent":
model = "agent"
if isinstance(instance, Agent):
obj = instance
else:
obj = None
else:
# ignore arg since it is invalid
return None
return instance_value
def replace_arg_db_values(
string: str, instance=None, shell: str = None, quotes=True # type:ignore
) -> Union[str, None]:
# resolve the value
value = get_db_value(string=string, instance=instance)
# check for model and property
if value is None:
DebugLog.error(
log_type=DebugLogType.SCRIPTING,
message=f"{instance} Not enough information to find value for: {string}. Only agent, site, client, and global are supported.",
message=f"Couldn't lookup value for: {string}. Make sure it exists",
)
return ""
if not obj:
return ""
# format args for str
if isinstance(value, str):
if shell == ScriptShell.POWERSHELL and "'" in value:
value = value.replace("'", "''")
# check if attr exists and isn't a function
if hasattr(obj, temp[1]) and not callable(getattr(obj, temp[1])):
temp1 = getattr(obj, temp[1])
if shell == ScriptShell.POWERSHELL and isinstance(temp1, str) and "'" in temp1:
temp1 = temp1.replace("'", "''")
return f"'{value}'" if quotes else value
value = f"'{temp1}'" if quotes else temp1
elif CustomField.objects.filter(model=model, name=temp[1]).exists():
field = CustomField.objects.get(model=model, name=temp[1])
model_fields = getattr(field, f"{model}_fields")
value = None
if model_fields.filter(**{model: obj}).exists():
if (
field.type != CustomFieldType.CHECKBOX
and model_fields.get(**{model: obj}).value
):
value = model_fields.get(**{model: obj}).value
elif field.type == CustomFieldType.CHECKBOX:
value = model_fields.get(**{model: obj}).value
# need explicit None check since a false boolean value will pass default value
if value is None and field.default_value is not None:
value = field.default_value
# check if value exists and if not use default
if value and field.type == CustomFieldType.MULTIPLE:
value = (
f"'{format_shell_array(value)}'"
if quotes
else format_shell_array(value)
)
elif value is not None and field.type == CustomFieldType.CHECKBOX:
value = format_shell_bool(value, shell)
else:
if (
shell == ScriptShell.POWERSHELL
and isinstance(value, str)
and "'" in value
):
value = value.replace("'", "''")
value = f"'{value}'" if quotes else value
else:
# ignore arg since property is invalid
DebugLog.error(
log_type=DebugLogType.SCRIPTING,
message=f"{instance} Couldn't find property on supplied variable: {string}. Make sure it exists as a custom field or a valid agent property",
# format args for list
elif isinstance(value, list):
return (
f"'{format_shell_array(value)}'"
if quotes
else format_shell_array(value)
)
return ""
# log any unhashable type errors
if value is not None:
return value
else:
DebugLog.error(
log_type=DebugLogType.SCRIPTING,
message=f" {instance}({instance.pk}) Couldn't lookup value for: {string}. Make sure it exists as a custom field or a valid agent property",
)
return ""
# format args for bool
elif value is True or value is False:
return format_shell_bool(value, shell)
def format_shell_array(value: list[str]) -> str:

2
docker/image-build.sh
View File

@@ -4,7 +4,7 @@ set -o errexit
set -o pipefail
# tactical tactical-frontend tactical-nats tactical-nginx tactical-meshcentral
DOCKER_IMAGES="tactical-nginx"
DOCKER_IMAGES="tactical tactical-frontend tactical-nats tactical-nginx tactical-meshcentral"
cd ..