Merge pull request #37 from NeedleInAJayStack/feature/nhaystack-basic

Adds nhaystack support using basic auth

Author: NeedleInAJayStack

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.0.16
+Upgrades to haystack v0.1.13 to loosen basic auth server requirements.
+
 ## 0.0.15
 Fixes haystack v0.1.12 upgrade.
 

go.mod

@@ -5,7 +5,7 @@ go 1.21
 require github.com/grafana/grafana-plugin-sdk-go v0.159.0
 
 require (
-	github.com/NeedleInAJayStack/haystack v0.1.12
+	github.com/NeedleInAJayStack/haystack v0.2.2
 	github.com/google/go-cmp v0.5.9
 )
 

go.sum

@@ -45,6 +45,14 @@ github.com/NeedleInAJayStack/haystack v0.1.11 h1:MPKSkIxmwLA7WZm2Vvul54bhaENLrok
 github.com/NeedleInAJayStack/haystack v0.1.11/go.mod h1:fTn/58sAzfySfXNkBku0DyEX0LXDOdcyIle5xglh3tA=
 github.com/NeedleInAJayStack/haystack v0.1.12 h1:Osz2uoxm52w+Fyx/s6mAsZcQ6DU6hxMPbwWYCsz3r3k=
 github.com/NeedleInAJayStack/haystack v0.1.12/go.mod h1:fTn/58sAzfySfXNkBku0DyEX0LXDOdcyIle5xglh3tA=
+github.com/NeedleInAJayStack/haystack v0.1.13 h1:VXUsnLPS1vRF5x85xGTXjjM1gf3rMjQwafPLYN6izqA=
+github.com/NeedleInAJayStack/haystack v0.1.13/go.mod h1:fTn/58sAzfySfXNkBku0DyEX0LXDOdcyIle5xglh3tA=
+github.com/NeedleInAJayStack/haystack v0.2.0 h1:zJE96QkPbKFBTQrwP23j0oksTUh0wbEq/rmk4+AaigM=
+github.com/NeedleInAJayStack/haystack v0.2.0/go.mod h1:fTn/58sAzfySfXNkBku0DyEX0LXDOdcyIle5xglh3tA=
+github.com/NeedleInAJayStack/haystack v0.2.1 h1:Hy+qWGzMW62DRYQoVEuIwQOgptk37Koy9ix2V39+PQY=
+github.com/NeedleInAJayStack/haystack v0.2.1/go.mod h1:fTn/58sAzfySfXNkBku0DyEX0LXDOdcyIle5xglh3tA=
+github.com/NeedleInAJayStack/haystack v0.2.2 h1:X7RGIF9y/ibmx07yPPKND8NyAjJSXzUN+IFaWG3Ha0s=
+github.com/NeedleInAJayStack/haystack v0.2.2/go.mod h1:fTn/58sAzfySfXNkBku0DyEX0LXDOdcyIle5xglh3tA=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=

pkg/plugin/datasource.go

@@ -149,7 +149,17 @@ func (datasource *Datasource) query(ctx context.Context, pCtx backend.PluginCont
 		}
 		return responseFromGrids([]haystack.Grid{eval})
 	case "hisRead":
-		hisRead, err := datasource.hisRead(haystack.NewRef(model.HisRead, ""), query.TimeRange)
+		refStr := model.HisRead
+		points, err := datasource.readById(refStr, variables)
+		if err != nil {
+			log.DefaultLogger.Error(err.Error())
+			return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("ReadById failure: %v", err.Error()))
+		}
+		if points.RowCount() < 1 {
+			return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("Id not found: %v", refStr))
+		}
+		point := points.RowAt(0)
+		hisRead, err := datasource.hisRead(point, query.TimeRange)
 		if err != nil {
 			log.DefaultLogger.Error(err.Error())
 			return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("HisRead failure: %v", err.Error()))
@@ -166,37 +176,31 @@ func (datasource *Datasource) query(ctx context.Context, pCtx backend.PluginCont
 		return response
 
 	case "hisReadFilter":
-		pointsGrid, readErr := datasource.read(model.HisReadFilter+" and hisStart", variables)
+		pointsGrid, readErr := datasource.read(model.HisReadFilter+" and his", variables)
 		if readErr != nil {
 			log.DefaultLogger.Error(readErr.Error())
 			return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("HisReadFilter failure: %v", readErr.Error()))
 		}
 		points := pointsGrid.Rows()
-		recLimit := 300
-		if len(points) > recLimit {
-			errMsg := fmt.Sprintf("Query exceeded record limit of %d: %d records", recLimit, len(points))
+		pointMax := 300
+		if len(points) == 0 {
+			errMsg := fmt.Sprintf("Query returned no historized records")
+			log.DefaultLogger.Error(errMsg)
+			return backend.ErrDataResponse(backend.StatusBadRequest, errMsg)
+		}
+		if len(points) > pointMax {
+			errMsg := fmt.Sprintf("Query exceeded record limit of %d: %d records", pointMax, len(points))
 			log.DefaultLogger.Error(errMsg)
 			return backend.ErrDataResponse(backend.StatusBadRequest, errMsg)
 		}
 
 		// Function to read a single point and send it to a channel.
 		readPoint := func(point haystack.Row, hisReadChannel chan haystack.Grid, wg *sync.WaitGroup) {
-			id := point.Get("id")
-			var ref haystack.Ref
-			switch id.(type) {
-			case haystack.Ref:
-				ref = id.(haystack.Ref)
-			default:
-				errMsg := fmt.Sprintf("id is not a ref: %v", id)
-				log.DefaultLogger.Error(errMsg)
-				hisReadChannel <- haystack.EmptyGrid()
-			}
-			hisRead, err := datasource.hisRead(ref, query.TimeRange)
+			hisRead, err := datasource.hisRead(point, query.TimeRange)
 			if err != nil {
 				log.DefaultLogger.Error(err.Error())
-				hisReadChannel <- haystack.EmptyGrid()
 			}
-			hisReadChannel <- hisRead
+			hisReadChannel <- hisRead // hisRead is empty under error condition
 			wg.Done()
 		}
 
@@ -310,9 +314,26 @@ func (datasource *Datasource) eval(expr string, variables map[string]string) (ha
 	)
 }
 
-func (datasource *Datasource) hisRead(id haystack.Ref, timeRange backend.TimeRange) (haystack.Grid, error) {
-	start := haystack.NewDateTimeFromGo(timeRange.From.UTC())
-	end := haystack.NewDateTimeFromGo(timeRange.To.UTC())
+func (datasource *Datasource) hisRead(point haystack.Row, timeRange backend.TimeRange) (haystack.Grid, error) {
+	id, idIsRef := point.Get("id").(haystack.Ref)
+	if !idIsRef {
+		return haystack.EmptyGrid(), fmt.Errorf("id is not a Ref")
+	}
+	tz, tzIsStr := point.Get("tz").(haystack.Str)
+	if !tzIsStr {
+		return haystack.EmptyGrid(), fmt.Errorf("tz is not a Str: %v", id)
+	}
+
+	// Must convert input date range to the point's timezone.
+	// See https://github.com/skyfoundry/haystack-java/blob/30380dbbe4b5d9be8eb3f400195b0cdcdcc67b95/src/main/java/org/projecthaystack/server/HServer.java#L328
+	start, startErr := haystack.NewDateTimeFromGo(timeRange.From).ToTz(tz.String())
+	if startErr != nil {
+		return haystack.EmptyGrid(), startErr
+	}
+	end, endErr := haystack.NewDateTimeFromGo(timeRange.To).ToTz(tz.String())
+	if endErr != nil {
+		return haystack.EmptyGrid(), endErr
+	}
 
 	return datasource.withRetry(
 		func() (haystack.Grid, error) {
@@ -333,6 +354,19 @@ func (datasource *Datasource) read(filter string, variables map[string]string) (
 	)
 }
 
+func (datasource *Datasource) readById(id string, variables map[string]string) (haystack.Grid, error) {
+	for name, val := range variables {
+		id = strings.ReplaceAll(id, name, val)
+	}
+
+	ref := haystack.NewRef(id, "")
+	return datasource.withRetry(
+		func() (haystack.Grid, error) {
+			return datasource.client.ReadByIds([]haystack.Ref{ref})
+		},
+	)
+}
+
 // nav returns the grid for the given navId, or the root nav if navId is nil
 // `navId` is expected to be a zinc-encoded Ref
 func (datasource *Datasource) nav(navId *string) (haystack.Grid, error) {
@@ -477,36 +511,35 @@ func dataFrameFromGrid(grid haystack.Grid) (*data.Frame, error) {
 
 		// Set Grafana field info from Haystack grid info
 		config := &data.FieldConfig{}
-		config.DisplayName = disFromGrid(grid, col)
+		config.DisplayName = disFromMeta(col.Meta(), col.Name())
 		config.Unit = unitFromGrid(grid, col)
 		field.Config = config
 		fields = append(fields, field)
 	}
 
 	frame := data.NewFrame("response", fields...)
-
-	switch id := grid.Meta().Get("id").(type) {
-	case haystack.Ref:
-		frame.Name = id.Dis()
-	default:
-		frame.Name = ""
-	}
+	frameName := disFromMeta(grid.Meta(), "")
+	frame.Name = frameName
 	return frame, nil
 }
 
-// disFromGrid returns the display name of a column in a haystack grid
-func disFromGrid(grid haystack.Grid, col haystack.Col) string {
-	// If column has 'id' meta, use the Ref name
-	id := col.Meta().Get("id")
-	if id != nil {
-		switch id := id.(type) {
-		case haystack.Ref:
-			return id.Dis()
-		default:
-			return col.Name()
+// disFromMeta returns the display name using metadata. It falls back to the provided string if no other name can be found
+func disFromMeta(meta haystack.Dict, name string) string {
+	// Use meta 'dis'
+	colDis, success := meta.Get("dis").(haystack.Str)
+	if success {
+		return colDis.String()
+	}
+	// Then 'id.dis' or 'id'
+	colRef, success := meta.Get("id").(haystack.Ref)
+	if success {
+		if colRef.Dis() != "" {
+			return colRef.Dis()
 		}
+		return colRef.ToZinc()
 	}
-	return col.Name()
+	// Then the input name. We shouldn't get here because all records should have an
+	return name
 }
 
 // unitFromGrid returns the unit of a column in a haystack grid

pkg/plugin/datasource_test.go

@@ -72,13 +72,19 @@ func TestQueryData_Eval_Dis(t *testing.T) {
 }
 
 func TestQueryData_HisRead(t *testing.T) {
-	response := haystack.NewGridBuilder()
-	response.AddCol("ts", map[string]haystack.Val{})
-	response.AddCol("v0", map[string]haystack.Val{})
-	response.AddRow([]haystack.Val{haystack.NewDateTimeFromGo(time.Unix(0, 0)), haystack.NewNumber(5, "kWh")})
+	readByIdsResponse := haystack.NewGridBuilder()
+	readByIdsResponse.AddCol("id", map[string]haystack.Val{})
+	readByIdsResponse.AddCol("tz", map[string]haystack.Val{})
+	readByIdsResponse.AddRow([]haystack.Val{haystack.NewRef("abcdefg-12345678", ""), haystack.NewStr("UTC")})
+
+	hisReadResponse := haystack.NewGridBuilder()
+	hisReadResponse.AddCol("ts", map[string]haystack.Val{})
+	hisReadResponse.AddCol("v0", map[string]haystack.Val{})
+	hisReadResponse.AddRow([]haystack.Val{haystack.NewDateTimeFromGo(time.Unix(0, 0)), haystack.NewNumber(5, "kWh")})
 
 	client := &testHaystackClient{
-		hisReadResponse: response.ToGrid(),
+		readByIdsResponse: readByIdsResponse.ToGrid(),
+		hisReadResponse:   hisReadResponse.ToGrid(),
 	}
 
 	actual := getResponse(
@@ -244,10 +250,11 @@ func getResponse(
 
 // TestHaystackClient is a mock of the HaystackClient interface
 type testHaystackClient struct {
-	navResponse     haystack.Grid
-	evalResponse    haystack.Grid
-	hisReadResponse haystack.Grid
-	readResponse    haystack.Grid
+	navResponse       haystack.Grid
+	evalResponse      haystack.Grid
+	hisReadResponse   haystack.Grid
+	readResponse      haystack.Grid
+	readByIdsResponse haystack.Grid
 }
 
 // Open is a no-op
@@ -288,3 +295,8 @@ func (c *testHaystackClient) HisReadAbsDateTime(ref haystack.Ref, start haystack
 func (c *testHaystackClient) Read(query string) (haystack.Grid, error) {
 	return c.readResponse, nil
 }
+
+// Read returns the ReadByIdsResponse
+func (c *testHaystackClient) ReadByIds(refs []haystack.Ref) (haystack.Grid, error) {
+	return c.readByIdsResponse, nil
+}

pkg/plugin/haystackClient.go

@@ -13,5 +13,6 @@ type HaystackClient interface {
 	Eval(string) (haystack.Grid, error)
 	HisReadAbsDateTime(haystack.Ref, haystack.DateTime, haystack.DateTime) (haystack.Grid, error)
 	Read(string) (haystack.Grid, error)
+	ReadByIds([]haystack.Ref) (haystack.Grid, error)
 	Nav(haystack.Val) (haystack.Grid, error)
 }

src/README.md

@@ -24,7 +24,7 @@ to create a new Haystack datasource. Next, fill in the required information:
 - The root Haystack API URL. The URLs for some popular Haystack servers are listed below:
   - SkySpark: `http://<host>/api/<proj>/`
   - Haxall: `http://<host>/api/`
-  - NHaystack: `http://<host>/`
+  - NHaystack: `http://<host>/<name_of_nhaystack_service>/`
 - The username and password. It is best practice to create a dedicated user for the Grafana integration.
 
 Once complete, select `Save & Test`. If you get a green check mark, the connection was successful!
@@ -77,6 +77,18 @@ are combined with commas, (`red,blue`), but this may be customized using the
 
 [Standard grafana alerting](https://grafana.com/docs/grafana/latest/alerting/) is supported by this data source.
 
+## Haystack Server Configuration
+
+### NHaystack
+
+Follow the setup instructions in the [`nhaystack` README](https://github.com/ci-richard-mcelhinney/nhaystack#usage).
+
+Currently only `Basic Auth` connectivity to `nhaystack` is supported. Basic auth should only be enabled when Niagara web traffic is encrypted using HTTPS.
+
+To add basic auth support, click and drag `baja/AuthenticationSchemes/WebServicesSchemes/HTTPBasicScheme` from the Palette to `Config/AuthenticationService/AuthenticationSchemes/` in the Nav pane. Then go to `Config/UserService/` in the Nav pane, create a user, and set the `Authentication Scheme Name` slot to `HTTPBasicScheme`. The user must also have an `Admin` role in order to access nhaystack endpoints.
+
+The root Haystack API URL is dependent on the name given to the nhaystack service: `http://<host>/<name_of_nhaystack_service>/`. This service name defaults to `haystack`, so unless renamed the URL is `http://<host>/haystack/`.
+
 ## Support
 
 You can view the code, contribute, or request support on this project's

src/datasource.ts

@@ -49,17 +49,29 @@ export class DataSource extends DataSourceWithBackend<HaystackQuery, HaystackDat
     let frame = result?.data?.find((frame: DataFrame) => {
       return frame.refId === refId;
     });
-    let opSymbols =
-      frame?.fields?.find((field: Field<any, Vector<string>>) => {
-        return field.name === 'def';
-      }).values ?? [];
-    let ops: string[] = opSymbols.map((opSymbol: string) => {
-      if (opSymbol.startsWith('^op:')) {
-        return opSymbol.substring(4);
-      } else {
-        return opSymbol;
+
+    let ops: string[] = []
+    
+    let defField = frame?.fields?.find((field: Field<any, Vector<string>>) => {
+      return field.name === 'def';
+    })
+    if (defField != null) {
+      ops = defField.values.map((opSymbol: string) => {
+        if (opSymbol.startsWith('^op:')) {
+          return opSymbol.substring(4);
+        } else {
+          return opSymbol;
+        }
+      });
+    } else {
+      // Include back-support for old `ops` format, which uses "name", not "defs". Used by nhaystack
+      let nameField = frame?.fields?.find((field: Field<any, Vector<string>>) => {
+        return field.name === 'name';
+      })
+      if (nameField != null) {
+        ops = nameField.values;
       }
-    });
+    }
 
     let availableQueryTypes = queryTypes.filter((queryType) => {
       return queryType.apiRequirements.every((apiRequirement) => {

src/types.ts

@@ -33,7 +33,7 @@ export interface HaystackVariableQuery {
 }
 
 export const DEFAULT_QUERY: Partial<HaystackQuery> = {
-  type: 'eval',
+  type: 'read',
   eval: '[{ts: $__timeRange_start, v0: 0}, {ts: $__timeRange_end, v0: 10}].toGrid',
   hisRead: 'abcdef-123456',
   hisReadFilter: 'point and his and temp and air and outside',