add robots.txt and sitemap.xml

src/http/handler.zig

@@ -56,6 +56,16 @@ pub fn handleWeather(
         res.body = @embedFile("favicon.ico");
         return;
     }
+    if (std.mem.eql(u8, "robots.txt", location)) {
+        res.content_type = .TEXT;
+        res.body = help.robots_txt;
+        return;
+    }
+    if (std.mem.eql(u8, "sitemap.xml", location)) {
+        res.header("Content-Type", "application/xml");
+        res.body = help.sitemap_xml;
+        return;
+    }
     log.debug("location = {s}, client_ip = {s}", .{ location, client_ip });
     if (location.len == 0) {
         res.content_type = .TEXT;
@@ -308,6 +318,44 @@ test "handler: favicon" {
     try ht.expectStatus(200);
 }
 
+test "handler: robots.txt" {
+    const allocator = std.testing.allocator;
+    const MockHarness = @import("Server.zig").MockHarness;
+
+    var harness = try MockHarness.init(allocator);
+    defer harness.deinit();
+
+    var ht = httpz.testing.init(.{});
+    defer ht.deinit();
+
+    ht.url("/robots.txt");
+    ht.param("location", "robots.txt");
+
+    try handleWeather(&harness.opts, ht.req, ht.res, "127.0.0.1");
+
+    try ht.expectStatus(200);
+    try ht.expectBody(help.robots_txt);
+}
+
+test "handler: sitemap.xml" {
+    const allocator = std.testing.allocator;
+    const MockHarness = @import("Server.zig").MockHarness;
+
+    var harness = try MockHarness.init(allocator);
+    defer harness.deinit();
+
+    var ht = httpz.testing.init(.{});
+    defer ht.deinit();
+
+    ht.url("/sitemap.xml");
+    ht.param("location", "sitemap.xml");
+
+    try handleWeather(&harness.opts, ht.req, ht.res, "127.0.0.1");
+
+    try ht.expectStatus(200);
+    try ht.expectBody(help.sitemap_xml);
+}
+
 test "handler: format j1 (json)" {
     const allocator = std.testing.allocator;
     const MockHarness = @import("Server.zig").MockHarness;

src/http/help.zig

@@ -126,6 +126,33 @@ pub const translation_page =
     \\
 ;
 
+pub const robots_txt =
+    \\User-agent: *
+    \\Disallow: /
+    \\Allow: /:help
+    \\Allow: /:translation
+    \\Allow: /robots.txt
+    \\Allow: /sitemap.xml
+    \\
+;
+
+pub const sitemap_xml =
+    \\<?xml version="1.0" encoding="UTF-8"?>
+    \\<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    \\  <url>
+    \\    <loc>/:help</loc>
+    \\    <changefreq>monthly</changefreq>
+    \\    <priority>1.0</priority>
+    \\  </url>
+    \\  <url>
+    \\    <loc>/:translation</loc>
+    \\    <changefreq>monthly</changefreq>
+    \\    <priority>0.8</priority>
+    \\  </url>
+    \\</urlset>
+    \\
+;
+
 test "help page exists" {
     try std.testing.expect(help_page.len > 0);
 }
@@ -133,3 +160,11 @@ test "help page exists" {
 test "translation page exists" {
     try std.testing.expect(translation_page.len > 0);
 }
+
+test "robots.txt exists" {
+    try std.testing.expect(robots_txt.len > 0);
+}
+
+test "sitemap.xml exists" {
+    try std.testing.expect(sitemap_xml.len > 0);
+}