prototype pagination for GET /api/v1/scenes. Add the ability to return metadata in a search query"

source/server/routes/api/v1/scenes/get.test.ts

@@ -14,17 +14,23 @@ import { read_cdh } from "../../../../utils/zip/index.js";
 
 
 describe("GET /api/v1/scenes", function(){
-  let vfs:Vfs, userManager:UserManager, ids :number[];
-  this.beforeEach(async function(){
+
+  let vfs:Vfs, userManager:UserManager, ids :number[], user :User, admin :User;
+  this.beforeAll(async function(){
     let locals = await createIntegrationContext(this);
     vfs = locals.vfs;
     userManager  = locals.userManager;
+
+    
+    user = await userManager.addUser("bob", "12345678", false);
+    admin = await userManager.addUser("alice", "12345678", true);
+
     ids = await Promise.all([
         vfs.createScene("foo"),
         vfs.createScene("bar"),
     ]);
   });
-  this.afterEach(async function(){
+  this.afterAll(async function(){
     await vfs.close();
     await fs.rm(this.dir, {recursive: true});
   });
@@ -40,7 +46,7 @@ describe("GET /api/v1/scenes", function(){
     let r = await request(this.server).get("/api/v1/scenes")
     .expect(200)
     .expect("Content-Type", "application/json; charset=utf-8");
-    expect(r.body).to.have.property("length", 2);
+    expect(r.body).to.have.property("scenes").to.have.property("length", 2);
   });
   
   it("can send a zip file", async function(){
@@ -68,19 +74,22 @@ describe("GET /api/v1/scenes", function(){
 
   describe("can get a list of scenes", function(){
     let scenes:number[];
-    this.beforeEach(async ()=>{
+    this.beforeAll(async ()=>{
       scenes = await Promise.all([
         vfs.createScene("s1"),
         vfs.createScene("s2"),
       ]);
     });
+    this.afterAll(async function(){
+      await Promise.all(scenes.map(id=>vfs.removeScene(id)));
+    });
 
     it("by name", async function(){
       let r = await request(this.server).get("/api/v1/scenes?name=s1&name=s2")
       .set("Accept", "application/json")
       .expect(200)
       .expect("Content-Type", "application/json; charset=utf-8");
-      expect(r.body).to.have.property("length", 2);
+      expect(r.body).to.have.property("scenes").to.have.property("length", 2);
     });
     
     it("by ids", async function(){
@@ -89,24 +98,27 @@ describe("GET /api/v1/scenes", function(){
       .send({scenes: scenes})
       .expect(200)
       .expect("Content-Type", "application/json; charset=utf-8");
-      expect(r.body).to.have.property("length", 2);
+      expect(r.body).to.have.property("scenes").to.have.property("length", 2);
     });
   });
 
   describe("can search scenes", async function(){
-    let scenes:number[], user :User, admin :User;
-    this.beforeEach(async ()=>{
-      user = await userManager.addUser("bob", "12345678", false);
-      admin = await userManager.addUser("alice", "12345678", true);
+    let scenes:number[];
+    this.beforeAll(async ()=>{
       scenes = await Promise.all([
         vfs.createScene("read", {[`${user.uid}`]:"read"}),
         vfs.createScene("write", {[`${user.uid}`]:"write"}),
+        await vfs.createScene("admin", {[`${user.uid}`]:"admin"}),
       ]);
     });
 
+    this.afterAll(async function(){
+      await Promise.all(scenes.map(id=>vfs.removeScene(id)));
+    });
+
+
     it("search by access level", async function(){
       let scene :any = await vfs.getScene("write", user.uid);
-
       delete scene.thumb;
 
       let r = await request(this.server).get(`/api/v1/scenes?access=write`)
@@ -115,7 +127,7 @@ describe("GET /api/v1/scenes", function(){
       .send({scenes: scenes})
       .expect(200)
       .expect("Content-Type", "application/json; charset=utf-8");
-      expect(r.body).to.deep.equal([
+      expect(r.body.scenes).to.deep.equal([
         {
           ...scene,
           mtime: scene.mtime.toISOString(),
@@ -125,7 +137,7 @@ describe("GET /api/v1/scenes", function(){
     });
 
     it("search by multiple access levels", async function(){
-      await vfs.createScene("admin", {[`${user.uid}`]:"admin"});
+
       let s1 :any = await vfs.getScene("write", user.uid);
       let s2 :any = await vfs.getScene("admin", user.uid);
 
@@ -138,7 +150,7 @@ describe("GET /api/v1/scenes", function(){
       .send({scenes: scenes})
       .expect(200)
       .expect("Content-Type", "application/json; charset=utf-8");
-      expect(r.body).to.deep.equal([
+      expect(r.body.scenes).to.deep.equal([
         {
           ...s2,
           mtime: s2.mtime.toISOString(),
@@ -167,7 +179,38 @@ describe("GET /api/v1/scenes", function(){
       .send({scenes: scenes})
       .expect(200)
       .expect("Content-Type", "application/json; charset=utf-8");
-      expect(r.body).to.deep.equal(scenes);
+      expect(r.body.scenes).to.deep.equal(scenes);
+    });
+  });
+
+  describe("supports pagination", function(){
+    let scenes:number[] = [];
+
+    this.beforeAll(async ()=>{
+      for(let i = 0; i < 110; i++){
+        scenes.push(await vfs.createScene(`scene_${i.toString(10).padStart(3, "0")}`));
+      }
+    });
+    
+    this.afterAll(async function(){
+      await Promise.all(scenes.map(id=>vfs.removeScene(id)));
+    });
+
+    it("use default limit", async function(){
+      let r = await request(this.server).get(`/api/v1/scenes`)
+      .set("Accept", "application/json")
+      .expect(200)
+      .expect("Content-Type", "application/json; charset=utf-8");
+      expect(r.body).to.have.property("scenes").to.have.property("length", 10);
+    });
+
+    it("use custom limit and offset", async function(){
+      let r = await request(this.server).get(`/api/v1/scenes?limit=12&offset=12&match=scene_`)
+      .set("Accept", "application/json")
+      .expect(200)
+      .expect("Content-Type", "application/json; charset=utf-8");
+      expect(r.body).to.have.property("scenes").to.have.property("length", 12);
+      expect(r.body.scenes[0]).to.have.property("name", "scene_012");
     });
   });
 

source/server/routes/api/v1/scenes/get.ts

@@ -11,7 +11,15 @@ import { ZipEntry, zip } from "../../../../utils/zip/index.js";
 export default async function getScenes(req :Request, res :Response){
   let vfs = getVfs(req);
   let u = getUser(req);
-  let {id: ids, name: names, match, access } = req.query;
+
+  let {
+    id: ids,
+    name: names,
+    match,
+    access,
+    limit,
+    offset
+  } = req.query;
 
   access = ((Array.isArray(access))?access : (access?[access]:undefined)) as any;
 
@@ -43,7 +51,12 @@ export default async function getScenes(req :Request, res :Response){
     scenes = await Promise.all(scenesList.map(name=>vfs.getScene(name)));
   }else{
     /**@fixme ugly hach to bypass permissions when not performing a search */
-    scenes = await vfs.getScenes((u.isAdministrator && !access && !match)?undefined: u.uid, {match: match as string, access: access as AccessType[]});
+    scenes = await vfs.getScenes((u.isAdministrator && !access && !match)?undefined: u.uid, {
+      match: match as string,
+      access: access as AccessType[],
+      limit: limit? parseInt(limit as string): undefined,
+      offset: offset? parseInt(offset as string): undefined,
+    });
   }
 
   //canonicalize scenes' thumb names
@@ -66,7 +79,7 @@ export default async function getScenes(req :Request, res :Response){
   }
   
   await wrapFormat(res, {
-    "application/json":()=>res.status(200).send(scenes),
+    "application/json":()=>res.status(200).send({scenes}),
 
     "text": ()=> res.status(200).send(scenes.map(m=>m.name).join("\n")+"\n"),
 

source/server/vfs/Scenes.ts

@@ -87,10 +87,12 @@ export default abstract class ScenesVfs extends BaseVfs{
    * get all scenes when called without params
    * Search scenes with structured queries when called with filters
    */
-  async getScenes(user_id ?:number, {access, match} :SceneQuery = {}) :Promise<Scene[]>{
+  async getScenes(user_id ?:number, {access, match, limit =10, offset = 0} :SceneQuery = {}) :Promise<Scene[]>{
     if(Array.isArray(access) && access.find(a=>AccessTypes.indexOf(a) === -1)){
       throw new BadRequestError(`Bad access type requested : ${access.join(", ")}`);
     }
+    if(typeof limit !="number" || Number.isNaN(limit) || limit < 0) throw new BadRequestError(`When provided, limit must be a number`);
+    if(typeof offset != "number" || Number.isNaN(offset) || offset < 0) throw new BadRequestError(`When provided, offset must be a number`);
     let with_filter = typeof user_id === "number" || match;
 
     if(match){
@@ -151,9 +153,12 @@ export default abstract class ScenesVfs extends BaseVfs{
       `: ""}
       GROUP BY scene_id
       ORDER BY LOWER(scene_name) ASC
+      LIMIT $offset, $limit
     `, {
       $user_id: user_id?.toString(10),
-      $match: match
+      $match: match,
+      $limit: Math.min(limit, 100),
+      $offset: offset,
     })).map(({ctime, mtime, id, access, ...m})=>({
       ...m,
       id,

source/server/vfs/types.ts

@@ -79,4 +79,6 @@ export interface SceneQuery {
   /** desired scene access level */
   access ?:AccessType[];
   match ?:string;
+  offset ?:number;
+  limit ?:number;
 }

source/server/vfs/vfs.test.ts

@@ -272,6 +272,45 @@ describe("Vfs", function(){
           expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1);
         });
       });
+
+      describe("pagination", function(){
+        it("rejects bad LIMIT", async function(){
+          let fixtures = [-1, "10", null];
+          for(let f of fixtures){
+            await expect(vfs.getScenes(0, {limit: f as any})).to.be.rejectedWith(BadRequestError);
+          }
+        });
+
+        it("rejects bad OFFSET", async function(){
+          let fixtures = [-1, "10", null];
+          for(let f of fixtures){
+            await expect(vfs.getScenes(0, {limit: f as any})).to.be.rejectedWith(BadRequestError);
+          }
+        });
+
+        it("respects pagination options", async function(){
+          for(let i = 0; i < 10; i++){
+            await vfs.createScene(`scene_${i}`);
+          }
+          let res = await vfs.getScenes(0, {limit: 1, offset: 0})
+          expect(res).to.have.property("length", 1);
+          expect(res[0]).to.have.property("name", "scene_0");
+
+          res = await vfs.getScenes(0, {limit: 2, offset: 2})
+          expect(res).to.have.property("length", 2);
+          expect(res[0]).to.have.property("name", "scene_2");
+          expect(res[1]).to.have.property("name", "scene_3");
+        });
+
+        it("limits LIMIT to 100", async function(){
+          for(let i = 0; i < 110; i++){
+            await vfs.createScene(`scene_${i}`);
+          }
+          let res = await vfs.getScenes(0, {limit: 110, offset: 0})
+          expect(res).to.have.property("length", 100);
+          expect(res[0]).to.have.property("name", "scene_0");
+        })
+      });
     });
 
     describe("createFolder(), removeFolder(), listFolders()", function(){

source/ui/state/withScenes.ts

@@ -18,6 +18,11 @@ export interface Scene {
     default :AccessType,
   }
 }
+
+export interface ApiResult {
+  scenes :Scene[];
+}
+
 export const AccessTypes = [
   null,
   "none",
@@ -58,9 +63,10 @@ export function withScenes<T extends Constructor<LitElement>>(baseClass:T) : T &
       let url = new URL("/api/v1/scenes", window.location.href);
       if(this.match) url.searchParams.set("match", this.match);
       if(this.access?.length) this.access.forEach(a=>url.searchParams.append("access", a));
+      url.searchParams.set("limit", "100");
       fetch(url, {signal: this.#loading.signal}).then(async (r)=>{
           if(!r.ok) throw new Error(`[${r.status}]: ${r.statusText}`);
-          this.list = (await r.json()) as Scene[];
+          this.list = ((await r.json()) as ApiResult).scenes;
       }).catch((e)=> {
           if(e.name == "AbortError") return;
           Notification.show(`Failed to fetch scenes list : ${e.message}`, "error");