Advanced Guide

How to Improve AI Readiness on Next.js

Full-stack React framework with native metadata and AI integration support. Follow these 7 steps to dramatically improve how AI agents discover and understand your Next.js site.

30-45 minutes

7 steps

Score: 35 → 92

Expected Score Improvement

Before

35/ 100

After

92/ 100

Estimated improvement based on implementing all steps in this guide. Actual results vary depending on existing site configuration and content quality.

Step-by-Step Instructions

Implement the Metadata API

Use Next.js App Router's built-in Metadata API to generate dynamic titles, descriptions, Open Graph tags, and canonical URLs for every page.

typescript

// app/layout.tsx — Global metadata defaults
import type { Metadata } from "next";

export const metadata: Metadata = {
  metadataBase: new URL("https://www.yoursite.com"),
  title: {
    default: "Your Site Name",
    template: "%s | Your Site Name",
  },
  description: "Your site description for AI agents and search engines",
  openGraph: {
    type: "website",
    siteName: "Your Site Name",
  },
  robots: {
    index: true,
    follow: true,
    googleBot: { index: true, follow: true },
  },
};

// app/blog/[slug]/page.tsx — Dynamic per-page metadata
export async function generateMetadata({ params }): Promise<Metadata> {
  const post = await getPost(params.slug);
  return {
    title: post.title,
    description: post.excerpt,
    openGraph: {
      title: post.title,
      description: post.excerpt,
      images: [post.ogImage],
    },
    alternates: {
      canonical: `/blog/${post.slug}`,
    },
  };
}

Generate llms.txt Programmatically

Create a route handler that dynamically generates llms.txt from your site's content and page structure.

typescript

// app/llms.txt/route.ts
import { NextResponse } from "next/server";

export async function GET() {
  const pages = await getAllPages(); // your data source

  const content = `# Your Site Name
# Brief description of what your site offers

## Main Sections
${pages
  .filter(p => p.isMainSection)
  .map(p => `- ${p.path}: ${p.title} - ${p.description}`)
  .join("\n")}

## Blog
${pages
  .filter(p => p.type === "blog")
  .slice(0, 20)
  .map(p => `- ${p.path}: ${p.title}`)
  .join("\n")}

## API Endpoints
- /api/v1: REST API documentation
- /.well-known/mcp.json: MCP endpoint configuration

## Contact
- Email: hello@yoursite.com
`;

  return new NextResponse(content, {
    headers: {
      "Content-Type": "text/plain; charset=utf-8",
      "Cache-Control": "public, max-age=3600, s-maxage=3600",
    },
  });
}

Set Up an MCP Endpoint

Create a Model Context Protocol (MCP) configuration endpoint that lets AI agents discover and interact with your site's capabilities.

typescript

// app/.well-known/mcp.json/route.ts
import { NextResponse } from "next/server";

export async function GET() {
  return NextResponse.json({
    schema_version: "1.0",
    name: "Your Site Name",
    description: "What your site offers to AI agents",
    capabilities: {
      search: {
        endpoint: "/api/search",
        description: "Search site content",
        parameters: {
          query: { type: "string", required: true },
          limit: { type: "number", default: 10 },
        },
      },
      content: {
        endpoint: "/api/content",
        description: "Get page content by URL",
        parameters: {
          url: { type: "string", required: true },
        },
      },
    },
  });
}

Configure Middleware for AI Crawlers

Use Next.js middleware to detect AI crawler user agents and serve optimized responses — ensuring they get full content without JavaScript rendering requirements.

typescript

// middleware.ts
import { NextResponse } from "next/server";
import type { NextRequest } from "next/server";

const AI_BOTS = [
  "GPTBot", "ChatGPT-User", "ClaudeBot",
  "PerplexityBot", "Amazonbot", "Google-Extended",
];

export function middleware(request: NextRequest) {
  const ua = request.headers.get("user-agent") || "";
  const isAIBot = AI_BOTS.some(bot => ua.includes(bot));

  const response = NextResponse.next();

  if (isAIBot) {
    // Add headers for AI crawlers
    response.headers.set("X-Robots-Tag", "index, follow");
    response.headers.set("Cache-Control", "public, max-age=3600");
  }

  return response;
}

export const config = {
  matcher: ["/((?!_next/static|_next/image|favicon.ico).*)"],
};

Add JSON-LD Structured Data

Create a reusable JSON-LD component and inject structured data into every page using Next.js's built-in script support.

typescript

// components/json-ld.tsx
export function JsonLd({ data }: { data: Record<string, unknown> }) {
  return (
    <script
      type="application/ld+json"
      dangerouslySetInnerHTML={{ __html: JSON.stringify(data) }}
    />
  );
}

// Usage in app/page.tsx:
import { JsonLd } from "@/components/json-ld";

export default function HomePage() {
  return (
    <>
      <JsonLd data={{
        "@context": "https://schema.org",
        "@type": "WebSite",
        name: "Your Site",
        url: "https://www.yoursite.com",
        potentialAction: {
          "@type": "SearchAction",
          target: "https://www.yoursite.com/search?q={search_term}",
          "query-input": "required name=search_term",
        },
      }} />
      <JsonLd data={{
        "@context": "https://schema.org",
        "@type": "Organization",
        name: "Your Business",
        url: "https://www.yoursite.com",
        logo: "https://www.yoursite.com/logo.png",
      }} />
      {/* Page content */}
    </>
  );
}

Generate Dynamic Sitemap

Use Next.js's built-in sitemap generation to create comprehensive, up-to-date sitemaps that include all your dynamic routes.

typescript

// app/sitemap.ts
import { MetadataRoute } from "next";

const BASE_URL = "https://www.yoursite.com";

export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
  // Static pages
  const staticPages = [
    { url: BASE_URL, lastModified: new Date(), priority: 1.0 },
    { url: `${BASE_URL}/about`, lastModified: new Date(), priority: 0.8 },
    { url: `${BASE_URL}/pricing`, lastModified: new Date(), priority: 0.9 },
  ];

  // Dynamic blog posts
  const posts = await getAllPosts();
  const blogPages = posts.map(post => ({
    url: `${BASE_URL}/blog/${post.slug}`,
    lastModified: new Date(post.updatedAt),
    priority: 0.7,
  }));

  return [...staticPages, ...blogPages];
}

// app/robots.ts
import { MetadataRoute } from "next";

export default function robots(): MetadataRoute.Robots {
  return {
    rules: [
      { userAgent: "*", allow: "/" },
      { userAgent: "GPTBot", allow: "/" },
      { userAgent: "ClaudeBot", allow: "/" },
      { userAgent: "PerplexityBot", allow: "/" },
    ],
    sitemap: "https://www.yoursite.com/sitemap.xml",
  };
}

Implement Server-Side Rendering for AI Content

Ensure critical content is server-rendered (not client-side only) so AI crawlers can access it without executing JavaScript.

typescript

// Use Server Components (default in App Router) for content pages
// app/blog/[slug]/page.tsx — This is a Server Component by default

export default async function BlogPost({ params }) {
  const post = await getPost(params.slug);

  return (
    <article>
      <header>
        <h1>{post.title}</h1>
        <time dateTime={post.date}>{post.formattedDate}</time>
        <p>{post.author}</p>
      </header>

      {/* Content is rendered server-side — AI crawlers see full HTML */}
      <section dangerouslySetInnerHTML={{ __html: post.content }} />

      <footer>
        <nav aria-label="Related posts">
          {post.relatedPosts.map(related => (
            <a key={related.slug} href={`/blog/${related.slug}`}>
              {related.title}
            </a>
          ))}
        </nav>
      </footer>
    </article>
  );
}

// AVOID: "use client" on content-heavy pages
// Client Components require JS execution — AI crawlers may not see content

Recommended Tools for Next.js

Next.js Metadata API
next-sitemap package
schema-dts (TypeScript schema types)
Vercel Analytics

Frequently Asked Questions

Does Next.js support llms.txt natively?

Not as a built-in feature, but it's trivial to implement. Create a route handler at app/llms.txt/route.ts that returns a text/plain response. You can dynamically generate the content from your page data, CMS, or database.

How do I add schema markup in Next.js App Router?

Use a script tag with type='application/ld+json' and dangerouslySetInnerHTML. Create a reusable JsonLd component for consistency. Place it in your page components or layout files. The Metadata API handles basic meta tags and Open Graph automatically.

Should I use SSR or SSG for AI readiness?

Both work well since AI crawlers receive server-rendered HTML in either case. SSG (Static Site Generation) is preferred for content that doesn't change frequently as it provides the fastest response times. Use ISR (Incremental Static Regeneration) for content that updates periodically.

What is MCP and do I need it for AI readiness?

MCP (Model Context Protocol) is a standard for AI agents to discover and interact with website capabilities. It's not required but gives you the highest AI readiness score. Implement it as a JSON endpoint at /.well-known/mcp.json describing your site's API capabilities.

How does Next.js compare to other CMS platforms for AI readiness?

Next.js offers the highest potential AI readiness score because you have full control over HTML output, metadata, route handlers, middleware, and server-side rendering. Unlike managed CMS platforms, there are no restrictions on file hosting, robots.txt, or custom endpoints.

Next.js Benchmark Data

See how Next.js sites score compared to other platforms, with protocol adoption rates and top-performing sites.

View Next.js AI Readiness Benchmark

Guides for Other Platforms

Ready to Check Your Next.js Site?

Run a free AI readiness scan to see your current score and get personalized recommendations for your Next.js site.

Scan Your Next.js Site

Step-by-Step Instructions

Implement the Metadata API

Use Next.js App Router's built-in Metadata API to generate dynamic titles, descriptions, Open Graph tags, and canonical URLs for every page.

typescript

// app/layout.tsx — Global metadata defaults
import type { Metadata } from "next";

export const metadata: Metadata = {
  metadataBase: new URL("https://www.yoursite.com"),
  title: {
    default: "Your Site Name",
    template: "%s | Your Site Name",
  },
  description: "Your site description for AI agents and search engines",
  openGraph: {
    type: "website",
    siteName: "Your Site Name",
  },
  robots: {
    index: true,
    follow: true,
    googleBot: { index: true, follow: true },
  },
};

// app/blog/[slug]/page.tsx — Dynamic per-page metadata
export async function generateMetadata({ params }): Promise<Metadata> {
  const post = await getPost(params.slug);
  return {
    title: post.title,
    description: post.excerpt,
    openGraph: {
      title: post.title,
      description: post.excerpt,
      images: [post.ogImage],
    },
    alternates: {
      canonical: `/blog/${post.slug}`,
    },
  };
}

Generate llms.txt Programmatically

Create a route handler that dynamically generates llms.txt from your site's content and page structure.

typescript

// app/llms.txt/route.ts
import { NextResponse } from "next/server";

export async function GET() {
  const pages = await getAllPages(); // your data source

  const content = `# Your Site Name
# Brief description of what your site offers

## Main Sections
${pages
  .filter(p => p.isMainSection)
  .map(p => `- ${p.path}: ${p.title} - ${p.description}`)
  .join("\n")}

## Blog
${pages
  .filter(p => p.type === "blog")
  .slice(0, 20)
  .map(p => `- ${p.path}: ${p.title}`)
  .join("\n")}

## API Endpoints
- /api/v1: REST API documentation
- /.well-known/mcp.json: MCP endpoint configuration

## Contact
- Email: hello@yoursite.com
`;

  return new NextResponse(content, {
    headers: {
      "Content-Type": "text/plain; charset=utf-8",
      "Cache-Control": "public, max-age=3600, s-maxage=3600",
    },
  });
}

Set Up an MCP Endpoint

Create a Model Context Protocol (MCP) configuration endpoint that lets AI agents discover and interact with your site's capabilities.

typescript

// app/.well-known/mcp.json/route.ts
import { NextResponse } from "next/server";

export async function GET() {
  return NextResponse.json({
    schema_version: "1.0",
    name: "Your Site Name",
    description: "What your site offers to AI agents",
    capabilities: {
      search: {
        endpoint: "/api/search",
        description: "Search site content",
        parameters: {
          query: { type: "string", required: true },
          limit: { type: "number", default: 10 },
        },
      },
      content: {
        endpoint: "/api/content",
        description: "Get page content by URL",
        parameters: {
          url: { type: "string", required: true },
        },
      },
    },
  });
}

Configure Middleware for AI Crawlers

Use Next.js middleware to detect AI crawler user agents and serve optimized responses — ensuring they get full content without JavaScript rendering requirements.

typescript

// middleware.ts
import { NextResponse } from "next/server";
import type { NextRequest } from "next/server";

const AI_BOTS = [
  "GPTBot", "ChatGPT-User", "ClaudeBot",
  "PerplexityBot", "Amazonbot", "Google-Extended",
];

export function middleware(request: NextRequest) {
  const ua = request.headers.get("user-agent") || "";
  const isAIBot = AI_BOTS.some(bot => ua.includes(bot));

  const response = NextResponse.next();

  if (isAIBot) {
    // Add headers for AI crawlers
    response.headers.set("X-Robots-Tag", "index, follow");
    response.headers.set("Cache-Control", "public, max-age=3600");
  }

  return response;
}

export const config = {
  matcher: ["/((?!_next/static|_next/image|favicon.ico).*)"],
};

Add JSON-LD Structured Data

Create a reusable JSON-LD component and inject structured data into every page using Next.js's built-in script support.

typescript

// components/json-ld.tsx
export function JsonLd({ data }: { data: Record<string, unknown> }) {
  return (
    <script
      type="application/ld+json"
      dangerouslySetInnerHTML={{ __html: JSON.stringify(data) }}
    />
  );
}

// Usage in app/page.tsx:
import { JsonLd } from "@/components/json-ld";

export default function HomePage() {
  return (
    <>
      <JsonLd data={{
        "@context": "https://schema.org",
        "@type": "WebSite",
        name: "Your Site",
        url: "https://www.yoursite.com",
        potentialAction: {
          "@type": "SearchAction",
          target: "https://www.yoursite.com/search?q={search_term}",
          "query-input": "required name=search_term",
        },
      }} />
      <JsonLd data={{
        "@context": "https://schema.org",
        "@type": "Organization",
        name: "Your Business",
        url: "https://www.yoursite.com",
        logo: "https://www.yoursite.com/logo.png",
      }} />
      {/* Page content */}
    </>
  );
}

Generate Dynamic Sitemap

Use Next.js's built-in sitemap generation to create comprehensive, up-to-date sitemaps that include all your dynamic routes.

typescript

// app/sitemap.ts
import { MetadataRoute } from "next";

const BASE_URL = "https://www.yoursite.com";

export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
  // Static pages
  const staticPages = [
    { url: BASE_URL, lastModified: new Date(), priority: 1.0 },
    { url: `${BASE_URL}/about`, lastModified: new Date(), priority: 0.8 },
    { url: `${BASE_URL}/pricing`, lastModified: new Date(), priority: 0.9 },
  ];

  // Dynamic blog posts
  const posts = await getAllPosts();
  const blogPages = posts.map(post => ({
    url: `${BASE_URL}/blog/${post.slug}`,
    lastModified: new Date(post.updatedAt),
    priority: 0.7,
  }));

  return [...staticPages, ...blogPages];
}

// app/robots.ts
import { MetadataRoute } from "next";

export default function robots(): MetadataRoute.Robots {
  return {
    rules: [
      { userAgent: "*", allow: "/" },
      { userAgent: "GPTBot", allow: "/" },
      { userAgent: "ClaudeBot", allow: "/" },
      { userAgent: "PerplexityBot", allow: "/" },
    ],
    sitemap: "https://www.yoursite.com/sitemap.xml",
  };
}

Implement Server-Side Rendering for AI Content

Ensure critical content is server-rendered (not client-side only) so AI crawlers can access it without executing JavaScript.

typescript

// Use Server Components (default in App Router) for content pages
// app/blog/[slug]/page.tsx — This is a Server Component by default

export default async function BlogPost({ params }) {
  const post = await getPost(params.slug);

  return (
    <article>
      <header>
        <h1>{post.title}</h1>
        <time dateTime={post.date}>{post.formattedDate}</time>
        <p>{post.author}</p>
      </header>

      {/* Content is rendered server-side — AI crawlers see full HTML */}
      <section dangerouslySetInnerHTML={{ __html: post.content }} />

      <footer>
        <nav aria-label="Related posts">
          {post.relatedPosts.map(related => (
            <a key={related.slug} href={`/blog/${related.slug}`}>
              {related.title}
            </a>
          ))}
        </nav>
      </footer>
    </article>
  );
}

// AVOID: "use client" on content-heavy pages
// Client Components require JS execution — AI crawlers may not see content