Add logs to Logs page (#151)

* Add websocket connection

* Add logs to the log page

* Make debug color more readable

* Remove TODO from front page

* Put all log entries in latest first order

* Add filtering of log entries by level

* Limit log entries and throttle updates

* Fix logger not throttling broadcasts

* Remove now unnecessary UI-side log throttling

* Filter incoming logs by log level

* Make log view more terminal-like
This commit is contained in:
WithoutPants 2019-10-25 10:07:07 +11:00 committed by Leopere
parent d7271d75fc
commit f29509577a
15 changed files with 519 additions and 19 deletions

View File

@ -0,0 +1,5 @@
fragment LogEntryData on LogEntry {
time
level
message
}

View File

@ -54,6 +54,11 @@ query Stats {
}
}
query Logs {
logs {
...LogEntryData
}
}
query Version {
version {
hash,

View File

@ -1,3 +1,9 @@
subscription MetadataUpdate {
metadataUpdate
}
subscription LoggingSubscribe {
loggingSubscribe {
...LogEntryData
}
}

View File

@ -37,6 +37,8 @@ type Query {
"""Organize scene markers by tag for a given scene ID"""
sceneMarkerTags(scene_id: ID!): [SceneMarkerTag!]!
logs: [LogEntry!]!
# Scrapers
"""Scrape a performer using Freeones"""
@ -101,6 +103,8 @@ type Mutation {
type Subscription {
"""Update from the metadata manager"""
metadataUpdate: String!
loggingSubscribe: [LogEntry!]!
}
schema {

View File

@ -0,0 +1,16 @@
"""Log entries"""
scalar Time
enum LogLevel {
Debug
Info
Progress
Warning
Error
}
type LogEntry {
time: Time!
level: LogLevel!
message: String!
}

View File

@ -0,0 +1,23 @@
package api
import (
"context"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
)
func (r *queryResolver) Logs(ctx context.Context) ([]*models.LogEntry, error) {
logCache := logger.GetLogCache()
ret := make([]*models.LogEntry, len(logCache))
for i, entry := range logCache {
ret[i] = &models.LogEntry{
Time: entry.Time,
Level: getLogLevel(entry.Type),
Message: entry.Message,
}
}
return ret, nil
}

View File

@ -0,0 +1,60 @@
package api
import (
"context"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
)
func getLogLevel(logType string) models.LogLevel {
if logType == "progress" {
return models.LogLevelProgress
} else if logType == "debug" {
return models.LogLevelDebug
} else if logType == "info" {
return models.LogLevelInfo
} else if logType == "warn" {
return models.LogLevelWarning
} else if logType == "error" {
return models.LogLevelError
}
// default to debug
return models.LogLevelDebug
}
func logEntriesFromLogItems(logItems []logger.LogItem) []*models.LogEntry {
ret := make([]*models.LogEntry, len(logItems))
for i, entry := range logItems {
ret[i] = &models.LogEntry{
Time: entry.Time,
Level: getLogLevel(entry.Type),
Message: entry.Message,
}
}
return ret
}
func (r *subscriptionResolver) LoggingSubscribe(ctx context.Context) (<-chan []*models.LogEntry, error) {
ret := make(chan []*models.LogEntry, 100)
stop := make(chan int, 1)
logSub := logger.SubscribeToLog(stop)
go func() {
for {
select {
case logEntries := <-logSub:
ret <- logEntriesFromLogItems(logEntries)
case <-ctx.Done():
stop <- 0
close(ret)
return
}
}
}()
return ret, nil
}

View File

@ -2,13 +2,16 @@ package logger
import (
"fmt"
"github.com/sirupsen/logrus"
"sync"
"time"
"github.com/sirupsen/logrus"
)
type LogItem struct {
Type string `json:"type"`
Message string `json:"message"`
Time time.Time `json:"time"`
Type string `json:"type"`
Message string `json:"message"`
}
var logger = logrus.New()
@ -16,14 +19,99 @@ var progressLogger = logrus.New()
var LogCache []LogItem
var mutex = &sync.Mutex{}
var logSubs []chan []LogItem
var waiting = false
var lastBroadcast = time.Now()
var logBuffer []LogItem
func addLogItem(l *LogItem) {
mutex.Lock()
l.Time = time.Now()
LogCache = append([]LogItem{*l}, LogCache...)
if len(LogCache) > 30 {
LogCache = LogCache[:len(LogCache)-1]
}
mutex.Unlock()
go broadcastLogItem(l)
}
func GetLogCache() []LogItem {
mutex.Lock()
ret := make([]LogItem, len(LogCache))
copy(ret, LogCache)
mutex.Unlock()
return ret
}
func SubscribeToLog(stop chan int) <-chan []LogItem {
ret := make(chan []LogItem, 100)
go func() {
<-stop
unsubscribeFromLog(ret)
}()
mutex.Lock()
logSubs = append(logSubs, ret)
mutex.Unlock()
return ret
}
func unsubscribeFromLog(toRemove chan []LogItem) {
mutex.Lock()
for i, c := range logSubs {
if c == toRemove {
logSubs = append(logSubs[:i], logSubs[i+1:]...)
}
}
close(toRemove)
mutex.Unlock()
}
func doBroadcastLogItems() {
// assumes mutex held
for _, c := range logSubs {
// don't block waiting to broadcast
select {
case c <- logBuffer:
default:
}
}
logBuffer = nil
waiting = false
lastBroadcast = time.Now()
}
func broadcastLogItem(l *LogItem) {
mutex.Lock()
logBuffer = append(logBuffer, *l)
// don't send more than once per second
if !waiting {
// if last broadcast was under a second ago, wait until a second has
// passed
timeSinceBroadcast := time.Since(lastBroadcast)
if timeSinceBroadcast.Seconds() < 1 {
waiting = true
time.AfterFunc(time.Second-timeSinceBroadcast, func() {
mutex.Lock()
doBroadcastLogItems()
mutex.Unlock()
})
} else {
doBroadcastLogItems()
}
}
// if waiting then adding it to the buffer is sufficient
mutex.Unlock()
}
func init() {

View File

@ -14,6 +14,7 @@
"@types/react-router-dom": "4.3.3",
"@types/video.js": "^7.2.11",
"apollo-boost": "0.4.0",
"apollo-link-ws": "^1.0.19",
"axios": "0.18.0",
"bulma": "0.7.5",
"formik": "1.5.7",
@ -32,6 +33,7 @@
"react-router-dom": "5.0.0",
"react-scripts": "3.0.1",
"react-use": "9.1.2",
"subscriptions-transport-ws": "^0.9.16",
"video.js": "^7.6.0"
},
"scripts": {

View File

@ -1,19 +1,188 @@
import {
H1,
H4,
H6,
Tag,
H4, FormGroup, HTMLSelect,
} from "@blueprintjs/core";
import React, { FunctionComponent } from "react";
import React, { FunctionComponent, useState, useEffect, useRef } from "react";
import * as GQL from "../../core/generated-graphql";
import { TextUtils } from "../../utils/text";
import { StashService } from "../../core/StashService";
interface IProps {}
function convertTime(logEntry : GQL.LogEntryDataFragment) {
function pad(val : number) {
var ret = val.toString();
if (val <= 9) {
ret = "0" + ret;
}
return ret;
}
var date = new Date(logEntry.time);
var month = date.getMonth() + 1;
var day = date.getDate();
var dateStr = date.getFullYear() + "-" + pad(month) + "-" + pad(day);
dateStr += " " + pad(date.getHours()) + ":" + pad(date.getMinutes()) + ":" + pad(date.getSeconds());
return dateStr;
}
class LogEntry {
public time: string;
public level: string;
public message: string;
public id: string;
private static nextId: number = 0;
public constructor(logEntry: GQL.LogEntryDataFragment) {
this.time = convertTime(logEntry);
this.level = logEntry.level;
this.message = logEntry.message;
var id = LogEntry.nextId++;
this.id = id.toString();
}
}
export const SettingsLogsPanel: FunctionComponent<IProps> = (props: IProps) => {
const { data, error } = StashService.useLoggingSubscribe();
const { data: existingData } = StashService.useLogs();
const logEntries = useRef<LogEntry[]>([]);
const [logLevel, setLogLevel] = useState<string>("Info");
const [filteredLogEntries, setFilteredLogEntries] = useState<LogEntry[]>([]);
const lastUpdate = useRef<number>(0);
const updateTimeout = useRef<NodeJS.Timeout>();
// maximum number of log entries to display. Subsequent entries will truncate
// the list, dropping off the oldest entries first.
const MAX_LOG_ENTRIES = 200;
function truncateLogEntries(entries : LogEntry[]) {
entries.length = Math.min(entries.length, MAX_LOG_ENTRIES);
}
function prependLogEntries(toPrepend : LogEntry[]) {
var newLogEntries = toPrepend.concat(logEntries.current);
truncateLogEntries(newLogEntries);
logEntries.current = newLogEntries;
}
function appendLogEntries(toAppend : LogEntry[]) {
var newLogEntries = logEntries.current.concat(toAppend);
truncateLogEntries(newLogEntries);
logEntries.current = newLogEntries;
}
useEffect(() => {
if (!data) { return; }
// append data to the logEntries
var convertedData = data.loggingSubscribe.map(convertLogEntry);
// filter subscribed data as it comes in, otherwise we'll end up
// truncating stuff that wasn't filtered out
convertedData = convertedData.filter(filterByLogLevel)
// put newest entries at the top
convertedData.reverse();
prependLogEntries(convertedData);
updateFilteredEntries();
}, [data]);
useEffect(() => {
if (!existingData || !existingData.logs) { return; }
var convertedData = existingData.logs.map(convertLogEntry);
appendLogEntries(convertedData);
updateFilteredEntries();
}, [existingData]);
function updateFilteredEntries() {
if (!updateTimeout.current) {
console.log("Updating after timeout");
}
updateTimeout.current = undefined;
var filteredEntries = logEntries.current.filter(filterByLogLevel);
setFilteredLogEntries(filteredEntries);
lastUpdate.current = new Date().getTime();
}
useEffect(() => {
updateFilteredEntries();
}, [logLevel]);
function convertLogEntry(logEntry : GQL.LogEntryDataFragment) {
return new LogEntry(logEntry);
}
function levelClass(level : string) {
return level.toLowerCase().trim();
}
interface ILogElementProps {
logEntry : LogEntry
}
function LogElement(props : ILogElementProps) {
// pad to maximum length of level enum
var level = props.logEntry.level.padEnd(GQL.LogLevel.Progress.length);
return (
<>
<span>{props.logEntry.time}</span>&nbsp;
<span className={levelClass(props.logEntry.level)}>{level}</span>&nbsp;
<span>{props.logEntry.message}</span>
<br/>
</>
);
}
function maybeRenderError() {
if (error) {
return (
<>
<span className={"error"}>Error connecting to log server: {error.message}</span><br/>
</>
);
}
}
const logLevels = ["Debug", "Info", "Warning", "Error"];
function filterByLogLevel(logEntry : LogEntry) {
if (logLevel == "Debug") {
return true;
}
var logLevelIndex = logLevels.indexOf(logLevel);
var levelIndex = logLevels.indexOf(logEntry.level);
return levelIndex >= logLevelIndex;
}
return (
<>
Logs
<H4>Logs</H4>
<div>
<FormGroup inline={true} label="Log Level">
<HTMLSelect
options={logLevels}
onChange={(event) => setLogLevel(event.target.value)}
value={logLevel}
/>
</FormGroup>
</div>
<div className="logs">
{maybeRenderError()}
{filteredLogEntries.map((logEntry) =>
<LogElement logEntry={logEntry} key={logEntry.id}/>
)}
</div>
</>
);
};

View File

@ -7,10 +7,8 @@ import {
H4,
} from "@blueprintjs/core";
import React, { FunctionComponent, useState } from "react";
import * as GQL from "../../../core/generated-graphql";
import { StashService } from "../../../core/StashService";
import { ErrorUtils } from "../../../utils/errors";
import { TextUtils } from "../../../utils/text";
import { ToastUtils } from "../../../utils/toasts";
import { GenerateButton } from "./GenerateButton";

View File

@ -56,8 +56,6 @@ export const Stats: FunctionComponent = () => {
* Filters for performers and studios only supports one item, even though it's a multi select.
TODO:
* Websocket connection to display logs in the UI
`}
</pre>
</div>

View File

@ -1,24 +1,58 @@
import ApolloClient from "apollo-boost";
import ApolloClient from "apollo-client";
import { WebSocketLink } from 'apollo-link-ws';
import { InMemoryCache } from 'apollo-cache-inmemory';
import { HttpLink, split } from "apollo-boost";
import _ from "lodash";
import { ListFilterModel } from "../models/list-filter/filter";
import * as GQL from "./generated-graphql";
import { SubscriptionHookOptions } from "react-apollo-hooks";
import { getMainDefinition } from "apollo-utilities";
import { platform } from "os";
export class StashService {
public static client: ApolloClient<any>;
public static initialize() {
const platformUrl = new URL(window.location.origin);
const wsPlatformUrl = new URL(window.location.origin);
wsPlatformUrl.protocol = "ws:";
if (!process.env.NODE_ENV || process.env.NODE_ENV === "development") {
platformUrl.port = "9999"; // TODO: Hack. Development expects port 9999
wsPlatformUrl.port = "9999";
if (process.env.REACT_APP_HTTPS === "true") {
platformUrl.protocol = "https:";
wsPlatformUrl.protocol = "wss:";
}
}
const url = platformUrl.toString().slice(0, -1);
const url = platformUrl.toString().slice(0, -1) + "/graphql";
const wsUrl = wsPlatformUrl.toString().slice(0, -1) + "/graphql";
const httpLink = new HttpLink({
uri: url,
});
const wsLink = new WebSocketLink({
uri: wsUrl,
options: {
reconnect: true
},
});
const link = split(
({ query }) => {
const { kind, operation } = getMainDefinition(query);
return kind === 'OperationDefinition' && operation === 'subscription';
},
wsLink,
httpLink,
);
const cache = new InMemoryCache();
StashService.client = new ApolloClient({
uri: `${url}/graphql`,
link: link,
cache: cache
});
(window as any).StashService = StashService;
@ -174,6 +208,20 @@ export class StashService {
return GQL.useConfigureInterface({ variables: { input }, refetchQueries: ["Configuration"] });
}
public static useMetadataUpdate() {
return GQL.useMetadataUpdate();
}
public static useLoggingSubscribe() {
return GQL.useLoggingSubscribe();
}
public static useLogs() {
return GQL.useLogs({
fetchPolicy: 'no-cache'
});
}
public static queryScrapeFreeones(performerName: string) {
return StashService.client.query<GQL.ScrapeFreeonesQuery>({
query: GQL.ScrapeFreeonesDocument,

View File

@ -172,6 +172,37 @@ video.preview {
white-space: pre-line;
}
.logs {
white-space: pre-wrap;
word-break: break-all;
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace;
font-size: smaller;
padding-right: 10px;
overflow-y: auto;
max-height: 100vh;
width: 120ch;
.debug {
color: lightgreen;
font-weight: bold;
}
.info {
color: white;
font-weight: bold;
}
.warning {
color: orange;
font-weight: bold;
}
.error {
color: red;
font-weight: bold;
}
}
span.block {
display: block;
}

View File

@ -2232,6 +2232,14 @@ apollo-link-http@^1.3.1:
apollo-link "^1.2.8"
apollo-link-http-common "^0.2.10"
apollo-link-ws@^1.0.19:
version "1.0.19"
resolved "https://registry.yarnpkg.com/apollo-link-ws/-/apollo-link-ws-1.0.19.tgz#dfa871d4df883a8777c9556c872fc892e103daa5"
integrity sha512-mRXmeUkc55ixOdYRtfq5rq3o9sboKghKABKroDVhJnkdS56zthBEWMAD+phajujOUbqByxjok0te8ABqByBdeQ==
dependencies:
apollo-link "^1.2.13"
tslib "^1.9.3"
apollo-link@^1.0.0, apollo-link@^1.0.6, apollo-link@^1.2.3, apollo-link@^1.2.8:
version "1.2.8"
resolved "https://registry.yarnpkg.com/apollo-link/-/apollo-link-1.2.8.tgz#0f252adefd5047ac1a9f35ba9439d216587dcd84"
@ -2239,6 +2247,16 @@ apollo-link@^1.0.0, apollo-link@^1.0.6, apollo-link@^1.2.3, apollo-link@^1.2.8:
dependencies:
zen-observable-ts "^0.8.15"
apollo-link@^1.2.13:
version "1.2.13"
resolved "https://registry.yarnpkg.com/apollo-link/-/apollo-link-1.2.13.tgz#dff00fbf19dfcd90fddbc14b6a3f9a771acac6c4"
integrity sha512-+iBMcYeevMm1JpYgwDEIDt/y0BB7VWyvlm/7x+TIPNLHCTCMgcEgDuW5kH86iQZWo0I7mNwQiTOz+/3ShPFmBw==
dependencies:
apollo-utilities "^1.3.0"
ts-invariant "^0.4.0"
tslib "^1.9.3"
zen-observable-ts "^0.8.20"
apollo-utilities@1.3.0, apollo-utilities@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/apollo-utilities/-/apollo-utilities-1.3.0.tgz#9803724c07ac94ca11dc26397edb58735d2b0211"
@ -2651,6 +2669,11 @@ babylon@^6.18.0:
resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3"
integrity sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==
backo2@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/backo2/-/backo2-1.0.2.tgz#31ab1ac8b129363463e35b3ebb69f4dfcfba7947"
integrity sha1-MasayLEpNjRj41s+u2n038+6eUc=
bail@^1.0.0:
version "1.0.3"
resolved "https://registry.yarnpkg.com/bail/-/bail-1.0.3.tgz#63cfb9ddbac829b02a3128cd53224be78e6c21a3"
@ -4754,6 +4777,11 @@ eventemitter3@^3.0.0:
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.0.tgz#090b4d6cdbd645ed10bf750d4b5407942d7ba163"
integrity sha512-ivIvhpq/Y0uSjcHDcOIccjmYjGLcP09MFGE7ysAwkAvkXfpZlC985pH2/ui64DKazbTW/4kN3yqozUxlXzI6cA==
eventemitter3@^3.1.0:
version "3.1.2"
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7"
integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==
events@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/events/-/events-3.0.0.tgz#9a0a0dfaf62893d92b875b8f2698ca4114973e88"
@ -6658,7 +6686,7 @@ istanbul-reports@^2.1.1:
dependencies:
handlebars "^4.1.2"
iterall@^1.1.3, iterall@^1.2.2:
iterall@^1.1.3, iterall@^1.2.1, iterall@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.2.2.tgz#92d70deb8028e0c39ff3164fdbf4d8b088130cd7"
integrity sha512-yynBb1g+RFUPY64fTrFv7nsjRrENBQJaX2UL+2Szc9REFrSNm1rpSXHGzhmAy7a9uv3vlvgBlXnf9RqmPH1/DA==
@ -11395,6 +11423,17 @@ stylis@3.5.0:
resolved "https://registry.yarnpkg.com/stylis/-/stylis-3.5.0.tgz#016fa239663d77f868fef5b67cf201c4b7c701e1"
integrity sha512-pP7yXN6dwMzAR29Q0mBrabPCe0/mNO1MSr93bhay+hcZondvMMTpeGyd8nbhYJdyperNT2DRxONQuUGcJr5iPw==
subscriptions-transport-ws@^0.9.16:
version "0.9.16"
resolved "https://registry.yarnpkg.com/subscriptions-transport-ws/-/subscriptions-transport-ws-0.9.16.tgz#90a422f0771d9c32069294c08608af2d47f596ec"
integrity sha512-pQdoU7nC+EpStXnCfh/+ho0zE0Z+ma+i7xvj7bkXKb1dvYHSZxgRPaU6spRP+Bjzow67c/rRDoix5RT0uU9omw==
dependencies:
backo2 "^1.0.2"
eventemitter3 "^3.1.0"
iterall "^1.2.1"
symbol-observable "^1.0.4"
ws "^5.2.0"
supports-color@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7"
@ -11442,7 +11481,7 @@ swap-case@^1.1.0:
lower-case "^1.1.1"
upper-case "^1.1.1"
symbol-observable@^1.0.2, symbol-observable@^1.1.0:
symbol-observable@^1.0.2, symbol-observable@^1.0.4, symbol-observable@^1.1.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804"
integrity sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==
@ -12766,6 +12805,14 @@ zen-observable-ts@^0.8.15:
dependencies:
zen-observable "^0.8.0"
zen-observable-ts@^0.8.20:
version "0.8.20"
resolved "https://registry.yarnpkg.com/zen-observable-ts/-/zen-observable-ts-0.8.20.tgz#44091e335d3fcbc97f6497e63e7f57d5b516b163"
integrity sha512-2rkjiPALhOtRaDX6pWyNqK1fnP5KkJJybYebopNSn6wDG1lxBoFs2+nwwXKoA6glHIrtwrfBBy6da0stkKtTAA==
dependencies:
tslib "^1.9.3"
zen-observable "^0.8.0"
zen-observable@^0.8.0:
version "0.8.13"
resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.13.tgz#a9f1b9dbdfd2d60a08761ceac6a861427d44ae2e"