Internal content linking implementation on website

Building an Internal Content Linking System

Internal links pass link authority between pages, help crawlers discover new content, and show users related materials. Manual linking doesn't scale with hundreds of articles—an automatic system is needed.

Architecture

Two approaches to automatic linking:

Keyword matching — finds keywords in text and replaces the first occurrence with a link.

Semantic matching — uses vector embeddings to find semantically similar pages.

Keyword-Based Linking

class AutoLinker
{
    // Dictionary: keyword → URL
    private array $linkMap;

    public function __construct()
    {
        // Load from cache or DB
        $this->linkMap = Cache::remember('autolink_map', 3600, function () {
            return Article::where('is_published', true)
                ->get()
                ->flatMap(fn($a) => collect($a->keywords)->mapWithKeys(
                    fn($kw) => [$kw => route('articles.show', $a->slug)]
                ))
                ->all();
        });

        // Sort by keyword length (longest first, to avoid partial matches)
        uksort($this->linkMap, fn($a, $b) => strlen($b) - strlen($a));
    }

    public function process(string $html, string $currentUrl): string
    {
        $dom = new \DOMDocument();
        @$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'));

        $linked = []; // Don't add the same link twice

        foreach ($this->linkMap as $keyword => $url) {
            if ($url === $currentUrl) continue;
            if (isset($linked[$url])) continue;

            // Search only in text nodes, not inside existing links
            $xpath = new \DOMXPath($dom);
            $textNodes = $xpath->query('//text()[not(ancestor::a) and not(ancestor::code) and not(ancestor::pre)]');

            foreach ($textNodes as $node) {
                $pattern = '/\b' . preg_quote($keyword, '/') . '\b/ui';
                if (preg_match($pattern, $node->nodeValue)) {
                    // Replace only first occurrence
                    $new = preg_replace($pattern,
                        "<a href=\"{$url}\">{$keyword}</a>",
                        $node->nodeValue, 1
                    );

                    $fragment = $dom->createDocumentFragment();
                    @$fragment->appendXML($new);
                    $node->parentNode->replaceChild($fragment, $node);

                    $linked[$url] = true;
                    break;
                }
            }
        }

        return $dom->saveHTML();
    }
}

Semantic Linking with Vector Embeddings

class SemanticLinker
{
    public function findRelated(Article $article, int $limit = 5): Collection
    {
        // Pre-calculated embeddings stored in PostgreSQL + pgvector
        return Article::selectRaw('*, embedding <=> ? AS distance', [$article->embedding])
            ->where('id', '!=', $article->id)
            ->where('is_published', true)
            ->whereRaw('embedding IS NOT NULL')
            ->orderBy('distance')
            ->limit($limit)
            ->get();
    }

    // Calculate embedding when saving article
    public function generateEmbedding(Article $article): void
    {
        $text = $article->title . "\n" . strip_tags($article->excerpt);

        $response = Http::withToken(config('openai.key'))
            ->post('https://api.openai.com/v1/embeddings', [
                'model' => 'text-embedding-3-small',
                'input' => $text,
            ]);

        $embedding = $response->json('data.0.embedding');
        $article->update(['embedding' => json_encode($embedding)]);
    }
}

"Related Articles" Component

// RelatedArticles.tsx
interface Article {
  id: number;
  title: string;
  slug: string;
  excerpt: string;
  category: string;
}

export function RelatedArticles({ articleId }: { articleId: number }) {
  const { data: related } = useQuery({
    queryKey: ['related', articleId],
    queryFn:  () => fetch(`/api/articles/${articleId}/related`).then(r => r.json()),
    staleTime: 5 * 60 * 1000,
  });

  if (!related?.length) return null;

  return (
    <aside className="mt-12 border-t pt-8">
      <h3 className="text-lg font-semibold mb-4">Related</h3>
      <div className="grid grid-cols-1 sm:grid-cols-2 gap-4">
        {related.map((article: Article) => (
          <a key={article.id} href={`/articles/${article.slug}`}
            className="block p-4 border rounded-lg hover:border-blue-400 transition-colors">
            <span className="text-xs text-blue-600 uppercase tracking-wide">{article.category}</span>
            <h4 className="font-medium mt-1 text-sm leading-snug">{article.title}</h4>
          </a>
        ))}
      </div>
    </aside>
  );
}

Internal Linking Status Report

-- Pages without incoming internal links (orphan pages)
SELECT a.title, a.slug
FROM articles a
WHERE a.is_published = true
  AND NOT EXISTS (
    SELECT 1 FROM article_links al WHERE al.target_id = a.id
  );

-- Pages with most incoming links
SELECT a.title, COUNT(al.id) AS incoming_links
FROM articles a
JOIN article_links al ON al.target_id = a.id
GROUP BY a.id, a.title
ORDER BY incoming_links DESC
LIMIT 20;

Timeframe

Auto-linking system (keyword + semantic) with related articles component: 3–5 business days.