liwenyun
/
CLIP
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update1

This commit is contained in:
GitJournal 2024-01-07 23:46:31 +08:00
parent a1d071733d
commit e5c9bc7fe9
41 changed files with 5382 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.fdignore Normal file
View File

@ -0,0 +1,6 @@
docs
*.{png,svg,in,tiktoken,npz,flac,json,ipynb}
*.{gz,tar,zip,rar,7z,xz}
LICENSE*
.*
*/.*

1
data/.fdignore Normal file
View File

@ -0,0 +1 @@
prompts.md

4
docs/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
!.gitignore
!*
!*/*
cache_db.json

522
docs/codeview.html Normal file
View File

@ -0,0 +1,522 @@
<!DOCTYPE html>
<html lang="en">
<!-- visit with anchor: #mycode.12-14 -->
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="icon"
href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' fill='currentColor' class='bi bi-code-square' viewBox='0 0 16 16'%3E%3Cpath d='M14 1a1 1 0 0 1 1 1v12a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zM2 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V2a2 2 0 0 0-2-2z'/%3E%3Cpath d='M6.854 4.646a.5.5 0 0 1 0 .708L4.207 8l2.647 2.646a.5.5 0 0 1-.708.708l-3-3a.5.5 0 0 1 0-.708l3-3a.5.5 0 0 1 .708 0zm2.292 0a.5.5 0 0 0 0 .708L11.793 8l-2.647 2.646a.5.5 0 0 0 .708.708l3-3a.5.5 0 0 0 0-.708l-3-3a.5.5 0 0 0-.708 0z'/%3E%3C/svg%3E"
type="image/svg+xml">
<title>Code View</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-python.min.js"></script>
<link rel="stylesheet"
href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/plugins/line-highlight/prism-line-highlight.css">
<script>
(function () {
if (typeof Prism === 'undefined' || typeof document === 'undefined' || !document.querySelector) {
return;
}
var LINE_NUMBERS_CLASS = 'line-numbers';
var LINKABLE_LINE_NUMBERS_CLASS = 'linkable-line-numbers';
var NEW_LINE_EXP = /\n(?!$)/g;
/**
* @param {string} selector
* @param {ParentNode} [container]
* @returns {HTMLElement[]}
*/
function $$(selector, container) {
return Array.prototype.slice.call((container || document).querySelectorAll(selector));
}
/**
* Returns whether the given element has the given class.
*
* @param {Element} element
* @param {string} className
* @returns {boolean}
*/
function hasClass(element, className) {
return element.classList.contains(className);
}
/**
* Calls the given function.
*
* @param {() => any} func
* @returns {void}
*/
function callFunction(func) {
func();
}
// Some browsers round the line-height, others don't.
// We need to test for it to position the elements properly.
var isLineHeightRounded = (function () {
var res;
return function () {
if (typeof res === 'undefined') {
var d = document.createElement('div');
d.style.fontSize = '13px'; // larger.
d.style.lineHeight = '1.5';
d.style.padding = '0';
d.style.border = '0';
d.innerHTML = '&nbsp;<br />&nbsp;';
document.body.appendChild(d);
// Browsers that round the line-height should have offsetHeight === 38
// The others should have 39.
res = d.offsetHeight === 38;
document.body.removeChild(d);
}
return res;
};
}());
/**
* Returns the top offset of the content box of the given parent and the content box of one of its children.
*
* @param {HTMLElement} parent
* @param {HTMLElement} child
*/
function getContentBoxTopOffset(parent, child) {
var parentStyle = getComputedStyle(parent);
var childStyle = getComputedStyle(child);
/**
* Returns the numeric value of the given pixel value.
*
* @param {string} px
*/
function pxToNumber(px) {
return +px.substr(0, px.length - 2);
}
return child.offsetTop
+ pxToNumber(childStyle.borderTopWidth)
+ pxToNumber(childStyle.paddingTop)
- pxToNumber(parentStyle.paddingTop);
}
/**
* Returns whether the Line Highlight plugin is active for the given element.
*
* If this function returns `false`, do not call `highlightLines` for the given element.
*
* @param {HTMLElement | null | undefined} pre
* @returns {boolean}
*/
function isActiveFor(pre) {
if (!pre || !/pre/i.test(pre.nodeName)) {
return false;
}
if (pre.hasAttribute('data-line')) {
return true;
}
if (pre.id && Prism.util.isActive(pre, LINKABLE_LINE_NUMBERS_CLASS)) {
// Technically, the line numbers plugin is also necessary but this plugin doesn't control the classes of
// the line numbers plugin, so we can't assume that they are present.
return true;
}
return false;
}
var scrollIntoView = true;
Prism.plugins.lineHighlight = {
/**
* Highlights the lines of the given pre.
*
* This function is split into a DOM measuring and mutate phase to improve performance.
* The returned function mutates the DOM when called.
*
* @param {HTMLElement} pre
* @param {string | null} [lines]
* @param {string} [classes='']
* @returns {() => void}
*/
highlightLines: function highlightLines(pre, lines, classes) {
lines = typeof lines === 'string' ? lines : (pre.getAttribute('data-line') || '');
var ranges = lines.replace(/\s+/g, '').split(',').filter(Boolean);
var offset = +pre.getAttribute('data-line-offset') || 0;
var parseMethod = isLineHeightRounded() ? parseInt : parseFloat;
var lineHeight = parseMethod(getComputedStyle(pre).lineHeight);
var hasLineNumbers = Prism.util.isActive(pre, LINE_NUMBERS_CLASS);
var codeElement = pre.querySelector('code');
var parentElement = hasLineNumbers ? pre : codeElement || pre;
var mutateActions = /** @type {(() => void)[]} */ ([]);
var lineBreakMatch = codeElement.textContent.match(NEW_LINE_EXP);
var numberOfLines = lineBreakMatch ? lineBreakMatch.length + 1 : 1;
/**
* The top offset between the content box of the <code> element and the content box of the parent element of
* the line highlight element (either `<pre>` or `<code>`).
*
* This offset might not be zero for some themes where the <code> element has a top margin. Some plugins
* (or users) might also add element above the <code> element. Because the line highlight is aligned relative
* to the <pre> element, we have to take this into account.
*
* This offset will be 0 if the parent element of the line highlight element is the `<code>` element.
*/
var codePreOffset = !codeElement || parentElement == codeElement ? 0 : getContentBoxTopOffset(pre, codeElement);
ranges.forEach(function (currentRange) {
var range = currentRange.split('-');
var start = +range[0];
var end = +range[1] || start;
end = Math.min(numberOfLines + offset, end);
if (end < start) {
return;
}
/** @type {HTMLElement} */
var line = pre.querySelector('.line-highlight[data-range="' + currentRange + '"]') || document.createElement('div');
mutateActions.push(function () {
line.setAttribute('aria-hidden', 'true');
line.setAttribute('data-range', currentRange);
line.className = (classes || '') + ' line-highlight';
});
// if the line-numbers plugin is enabled, then there is no reason for this plugin to display the line numbers
if (hasLineNumbers && Prism.plugins.lineNumbers) {
var startNode = Prism.plugins.lineNumbers.getLine(pre, start);
var endNode = Prism.plugins.lineNumbers.getLine(pre, end);
if (startNode) {
var top = startNode.offsetTop + codePreOffset + 'px';
mutateActions.push(function () {
line.style.top = top;
});
}
if (endNode) {
var height = (endNode.offsetTop - startNode.offsetTop) + endNode.offsetHeight + 'px';
mutateActions.push(function () {
line.style.height = height;
});
}
} else {
mutateActions.push(function () {
line.setAttribute('data-start', String(start));
if (end > start) {
line.setAttribute('data-end', String(end));
}
line.style.top = (start - offset - 1) * lineHeight + codePreOffset + 'px';
line.textContent = new Array(end - start + 2).join(' \n');
});
}
mutateActions.push(function () {
line.style.width = pre.scrollWidth + 'px';
});
mutateActions.push(function () {
// allow this to play nicely with the line-numbers plugin
// need to attack to pre as when line-numbers is enabled, the code tag is relatively which screws up the positioning
parentElement.appendChild(line);
});
});
var id = pre.id;
if (hasLineNumbers && Prism.util.isActive(pre, LINKABLE_LINE_NUMBERS_CLASS) && id) {
// This implements linkable line numbers. Linkable line numbers use Line Highlight to create a link to a
// specific line. For this to work, the pre element has to:
// 1) have line numbers,
// 2) have the `linkable-line-numbers` class or an ascendant that has that class, and
// 3) have an id.
if (!hasClass(pre, LINKABLE_LINE_NUMBERS_CLASS)) {
// add class to pre
mutateActions.push(function () {
pre.classList.add(LINKABLE_LINE_NUMBERS_CLASS);
});
}
var start = parseInt(pre.getAttribute('data-start') || '1');
// iterate all line number spans
$$('.line-numbers-rows > span', pre).forEach(function (lineSpan, i) {
var lineNumber = i + start;
lineSpan.onclick = function () {
var hash = id + '.' + lineNumber;
// this will prevent scrolling since the span is obviously in view
scrollIntoView = false;
location.hash = hash;
setTimeout(function () {
scrollIntoView = true;
}, 1);
};
});
}
return function () {
mutateActions.forEach(callFunction);
};
}
};
function applyHash() {
var hash = location.hash.slice(1);
// Remove pre-existing temporary lines
$$('.temporary.line-highlight').forEach(function (line) {
line.parentNode.removeChild(line);
});
var range = (hash.match(/\.([\d,-]+)$/) || [, ''])[1];
if (!range || document.getElementById(hash)) {
return;
}
var id = hash.slice(0, hash.lastIndexOf('.'));
var pre = document.getElementById(id);
if (!pre) {
return;
}
// now we have hash.
if (!pre.hasAttribute('data-line')) {
pre.setAttribute('data-line', '');
}
var mutateDom = Prism.plugins.lineHighlight.highlightLines(pre, range, 'temporary ');
mutateDom();
if (scrollIntoView) {
document.querySelector('.temporary.line-highlight').scrollIntoView();
}
}
var fakeTimer = 0; // Hack to limit the number of times applyHash() runs
Prism.hooks.add('before-sanity-check', function (env) {
var pre = env.element.parentElement;
if (!isActiveFor(pre)) {
return;
}
/*
* Cleanup for other plugins (e.g. autoloader).
*
* Sometimes <code> blocks are highlighted multiple times. It is necessary
* to cleanup any left-over tags, because the whitespace inside of the <div>
* tags change the content of the <code> tag.
*/
var num = 0;
$$('.line-highlight', pre).forEach(function (line) {
num += line.textContent.length;
line.parentNode.removeChild(line);
});
// Remove extra whitespace
if (num && /^(?: \n)+$/.test(env.code.slice(-num))) {
env.code = env.code.slice(0, -num);
}
});
Prism.hooks.add('complete', function completeHook(env) {
var pre = env.element.parentElement;
if (!isActiveFor(pre)) {
return;
}
clearTimeout(fakeTimer);
var hasLineNumbers = Prism.plugins.lineNumbers;
var isLineNumbersLoaded = env.plugins && env.plugins.lineNumbers;
if (hasClass(pre, LINE_NUMBERS_CLASS) && hasLineNumbers && !isLineNumbersLoaded) {
Prism.hooks.add('line-numbers', completeHook);
} else {
var mutateDom = Prism.plugins.lineHighlight.highlightLines(pre);
mutateDom();
fakeTimer = setTimeout(applyHash, 1);
}
});
window.addEventListener('hashchange', applyHash);
function conditionalApplyHash() {
const pre_elem = document.getElementById('mycode');
const loaded = pre_elem.getAttribute('data-src-status');
if (loaded == "loaded") {
applyHash();
} else {
setTimeout(conditionalApplyHash, 500)
}
}
function getQueryParams() {
var search = window.location.search.substring(1); // Remove leading '?'
var queryParams = {};
search.split('&').forEach(function (pair) {
var parts = pair.split('=');
var key = decodeURIComponent(parts[0]);
var value = decodeURIComponent(parts[1]);
queryParams[key] = value;
});
return queryParams;
}
function loadedAction() {
// console.log('location search:', window.location.search);
const section_elem = document.getElementById('code-section');
const pre_elem = document.createElement('pre');
const queryParams = getQueryParams(window.location.search);
// const queryParams = new URLSearchParams(window.location.search);
const language = queryParams.language;
const code_path = queryParams.file;
const project_name = queryParams.project;
const h1_element = document.getElementById('code-path');
// h1_element.textContent = code_path.slice('src/'.length);
// debugger;
h1_element.textContent = project_name + "/" + code_path.slice('src/'.length);
pre_elem.className = `language-${language}`
pre_elem.id = "mycode";
pre_elem.setAttribute("data-src", code_path);
section_elem.appendChild(pre_elem)
Prism.highlightElement(pre_elem);
conditionalApplyHash();
// applyHash();
// Prism.highlightElement(pre_elem, () => {applyHash()});
// Prism.highlightElement(pre_elem).then(applyHash);
}
// window.addEventListener('load', loadedAction);
document.addEventListener('DOMContentLoaded', loadedAction);
window.addEventListener('resize', function () {
var actions = $$('pre')
.filter(isActiveFor)
.map(function (pre) {
return Prism.plugins.lineHighlight.highlightLines(pre);
});
actions.forEach(callFunction);
});
}());
</script>
<style>
/*
html, body{
background-color: transparent;
} */
#code-path {
/* overflow-x:scroll; */
/* opt1 */
/* word-wrap: break-word;
overflow-wrap: break-word */
/* opt2 */
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
direction: rtl;
}
html,
body {
/* max-width: 900px; */
/* margin: 0 auto; */
/* margin-left: 5%;
margin-right: 5%; */
height: 100%;
display: flex;
flex-direction: column;
font-family: 'Roboto', sans-serif;
/* Adjust based on the height of the header */
}
section {
flex: 1;
/* margin: 0 auto; */
/* Provide spacing to accommodate the fixed header */
/* Ensure the section fills the remaining viewport height */
overflow-y: auto;
/* Enable vertical scrolling if content exceeds viewport height */
}
.container {
display: flex;
flex-direction: column;
align-items: left;
justify-content: center;
text-align: left;
}
.monospace-text {
font-family: "Courier New", monospace;
color: #333;
}
@media (max-width: 767px) {
.container {
padding-left: 10px;
}
html,
body {
margin-left: 3%;
margin-right: 3%;
}
#code-section {
font-size: 14px;
}
}
#code-section {
border: 1px solid #ccc;
}
/* Styles for desktop devices */
@media (min-width: 768px) {
.container {
padding-left: 1.6%;
}
html,
body {
margin-left: 5%;
margin-right: 5%;
}
#code-section {
font-size: 17px;
}
}
</style>
</head>
<body>
<header class="container" data-plugin-header="line-numbers">
<p id="code-path" class="monospace-text">Code Preview</p>
</header>
<section id="code-section">
<!-- <pre id="mycode" class="language-python" data-src="code_view_demo.py">
</pre> -->
</section>
</body>
</html>

544
docs/data/0.json Normal file
View File

@ -0,0 +1,544 @@
{
"0": {
"file_id": 0,
"content": "/README.md",
"type": "filepath"
},
"1": {
"file_id": 0,
"content": "Both comments discuss using CLIP, a pre-trained model, for image and text feature extraction. Comment A focuses on CIFAR100 feature extraction and similarity computation, while Comment B covers logistic regression implementation, hyperparameter tuning, and utilizes OpenCLIP and Hugging Face CLIP implementations.",
"type": "summary"
},
"2": {
"file_id": 0,
"content": "# CLIP\n[[Blog]](https://openai.com/blog/clip/) [[Paper]](https://arxiv.org/abs/2103.00020) [[Model Card]](model-card.md) [[Colab]](https://colab.research.google.com/github/openai/clip/blob/master/notebooks/Interacting_with_CLIP.ipynb)\nCLIP (Contrastive Language-Image Pre-Training) is a neural network trained on a variety of (image, text) pairs. It can be instructed in natural language to predict the most relevant text snippet, given an image, without directly optimizing for the task, similarly to the zero-shot capabilities of GPT-2 and 3. We found CLIP matches the performance of the original ResNet50 on ImageNet “zero-shot” without using any of the original 1.28M labeled examples, overcoming several major challenges in computer vision.\n## Approach\n![CLIP](CLIP.png)\n## Usage\nFirst, [install PyTorch 1.7.1](https://pytorch.org/get-started/locally/) (or later) and torchvision, as well as small additional dependencies, and then install this repo as a Python package. On a CUDA GPU machine, the following will do the trick:",
"type": "code",
"location": "/README.md:1-17"
},
"3": {
"file_id": 0,
"content": "The code provides a brief introduction to CLIP, a neural network trained on various image-text pairs. It can predict relevant text based on an image without directly optimizing for the task and matches the performance of ResNet50 on ImageNet \"zero-shot\" without using any labeled examples. The code also explains how to install necessary dependencies and set up the environment to use CLIP.",
"type": "comment"
},
"4": {
"file_id": 0,
"content": "```bash\n$ conda install --yes -c pytorch pytorch=1.7.1 torchvision cudatoolkit=11.0\n$ pip install ftfy regex tqdm\n$ pip install git+https://github.com/openai/CLIP.git\n```\nReplace `cudatoolkit=11.0` above with the appropriate CUDA version on your machine or `cpuonly` when installing on a machine without a GPU.\n```python\nimport torch\nimport clip\nfrom PIL import Image\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel, preprocess = clip.load(\"ViT-B/32\", device=device)\nimage = preprocess(Image.open(\"CLIP.png\")).unsqueeze(0).to(device)\ntext = clip.tokenize([\"a diagram\", \"a dog\", \"a cat\"]).to(device)\nwith torch.no_grad():\n image_features = model.encode_image(image)\n text_features = model.encode_text(text)\n logits_per_image, logits_per_text = model(image, text)\n probs = logits_per_image.softmax(dim=-1).cpu().numpy()\nprint(\"Label probs:\", probs) # prints: [[0.9927937 0.00421068 0.00299572]]\n```\n## API\nThe CLIP module `clip` provides the following methods:\n#### `clip.available_models()`\nReturns the names of the available CLIP models.",
"type": "code",
"location": "/README.md:19-55"
},
"5": {
"file_id": 0,
"content": "Code installs necessary packages for running CLIP and loads the model.",
"type": "comment"
},
"6": {
"file_id": 0,
"content": "#### `clip.load(name, device=..., jit=False)`\nReturns the model and the TorchVision transform needed by the model, specified by the model name returned by `clip.available_models()`. It will download the model as necessary. The `name` argument can also be a path to a local checkpoint.\nThe device to run the model can be optionally specified, and the default is to use the first CUDA device if there is any, otherwise the CPU. When `jit` is `False`, a non-JIT version of the model will be loaded.\n#### `clip.tokenize(text: Union[str, List[str]], context_length=77)`\nReturns a LongTensor containing tokenized sequences of given text input(s). This can be used as the input to the model\n---\nThe model returned by `clip.load()` supports the following methods:\n#### `model.encode_image(image: Tensor)`\nGiven a batch of images, returns the image features encoded by the vision portion of the CLIP model.\n#### `model.encode_text(text: Tensor)`\nGiven a batch of text tokens, returns the text features encoded by the language portion of the CLIP model.",
"type": "code",
"location": "/README.md:57-77"
},
"7": {
"file_id": 0,
"content": "This code snippet is for the CLIP library, which provides a model for visual-textual similarity. It includes two primary functions: `clip.load()` and `clip.tokenize()`. The `clip.load()` function loads a pre-trained CLIP model specified by the `name` parameter or downloads it if necessary. The `clip.tokenize()` function tokenizes input text(s) and returns LongTensor containing the tokenized sequences. The loaded model also supports two methods: `model.encode_image()` to encode image features and `model.encode_text()` to encode text features.",
"type": "comment"
},
"8": {
"file_id": 0,
"content": "#### `model(image: Tensor, text: Tensor)`\nGiven a batch of images and a batch of text tokens, returns two Tensors, containing the logit scores corresponding to each image and text input. The values are cosine similarities between the corresponding image and text features, times 100.\n## More Examples\n### Zero-Shot Prediction\nThe code below performs zero-shot prediction using CLIP, as shown in Appendix B in the paper. This example takes an image from the [CIFAR-100 dataset](https://www.cs.toronto.edu/~kriz/cifar.html), and predicts the most likely labels among the 100 textual labels from the dataset.\n```python\nimport os\nimport clip\nimport torch\nfrom torchvision.datasets import CIFAR100\n# Load the model\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel, preprocess = clip.load('ViT-B/32', device)\n# Download the dataset\ncifar100 = CIFAR100(root=os.path.expanduser(\"~/.cache\"), download=True, train=False)\n# Prepare the inputs\nimage, class_id = cifar100[3637]\nimage_input = preprocess(image).unsqueeze(0).to(device)",
"type": "code",
"location": "/README.md:79-106"
},
"9": {
"file_id": 0,
"content": "The code snippet loads the CLIP model (ViT-B/32) and prepares inputs for zero-shot prediction using an image from the CIFAR-100 dataset.",
"type": "comment"
},
"10": {
"file_id": 0,
"content": "text_inputs = torch.cat([clip.tokenize(f\"a photo of a {c}\") for c in cifar100.classes]).to(device)\n# Calculate features\nwith torch.no_grad():\n image_features = model.encode_image(image_input)\n text_features = model.encode_text(text_inputs)\n# Pick the top 5 most similar labels for the image\nimage_features /= image_features.norm(dim=-1, keepdim=True)\ntext_features /= text_features.norm(dim=-1, keepdim=True)\nsimilarity = (100.0 * image_features @ text_features.T).softmax(dim=-1)\nvalues, indices = similarity[0].topk(5)\n# Print the result\nprint(\"\\nTop predictions:\\n\")\nfor value, index in zip(values, indices):\n print(f\"{cifar100.classes[index]:>16s}: {100 * value.item():.2f}%\")\n```\nThe output will look like the following (the exact numbers may be slightly different depending on the compute device):\n```\nTop predictions:\n snake: 65.31%\n turtle: 12.29%\n sweet_pepper: 3.83%\n lizard: 1.88%\n crocodile: 1.75%\n```\nNote that this example uses the `encode_image()` and `encode_text()` methods that return the encoded features of given inputs.",
"type": "code",
"location": "/README.md:107-138"
},
"11": {
"file_id": 0,
"content": "This code calculates the similarity between image features and text features using dot product and softmax, then prints the top 5 most similar labels for the given image.",
"type": "comment"
},
"12": {
"file_id": 0,
"content": "### Linear-probe evaluation\nThe example below uses [scikit-learn](https://scikit-learn.org/) to perform logistic regression on image features.\n```python\nimport os\nimport clip\nimport torch\nimport numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom torch.utils.data import DataLoader\nfrom torchvision.datasets import CIFAR100\nfrom tqdm import tqdm\n# Load the model\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel, preprocess = clip.load('ViT-B/32', device)\n# Load the dataset\nroot = os.path.expanduser(\"~/.cache\")\ntrain = CIFAR100(root, download=True, train=True, transform=preprocess)\ntest = CIFAR100(root, download=True, train=False, transform=preprocess)\ndef get_features(dataset):\n all_features = []\n all_labels = []\n with torch.no_grad():\n for images, labels in tqdm(DataLoader(dataset, batch_size=100)):\n features = model.encode_image(images.to(device))\n all_features.append(features)\n all_labels.append(labels)\n return torch.cat(all_features).cpu().numpy(), torch.cat(all_labels).cpu().numpy()",
"type": "code",
"location": "/README.md:141-177"
},
"13": {
"file_id": 0,
"content": "This code loads a pre-trained CLIP model, then applies it to CIFAR100 dataset for feature extraction using logistic regression.",
"type": "comment"
},
"14": {
"file_id": 0,
"content": "# Calculate the image features\ntrain_features, train_labels = get_features(train)\ntest_features, test_labels = get_features(test)\n# Perform logistic regression\nclassifier = LogisticRegression(random_state=0, C=0.316, max_iter=1000, verbose=1)\nclassifier.fit(train_features, train_labels)\n# Evaluate using the logistic regression classifier\npredictions = classifier.predict(test_features)\naccuracy = np.mean((test_labels == predictions).astype(float)) * 100.\nprint(f\"Accuracy = {accuracy:.3f}\")\n```\nNote that the `C` value should be determined via a hyperparameter sweep using a validation split.\n## See Also\n* [OpenCLIP](https://github.com/mlfoundations/open_clip): includes larger and independently trained CLIP models up to ViT-G/14\n* [Hugging Face implementation of CLIP](https://huggingface.co/docs/transformers/model_doc/clip): for easier integration with the HF ecosystem",
"type": "code",
"location": "/README.md:179-199"
},
"15": {
"file_id": 0,
"content": "Calculating image features, performing logistic regression, and evaluating using a logistic regression classifier.\nHyperparameter C should be determined via validation split.\nSee also OpenCLIP and Hugging Face CLIP implementation.",
"type": "comment"
},
"16": {
"file_id": 1,
"content": "/hubconf.py",
"type": "filepath"
},
"17": {
"file_id": 1,
"content": "This code defines functions for creating entry points to load CLIP models and converting PIL images into tensors, while also mapping model names and updating the global namespace with different model entrypoints.",
"type": "summary"
},
"18": {
"file_id": 1,
"content": "from clip.clip import tokenize as _tokenize, load as _load, available_models as _available_models\nimport re\nimport string\ndependencies = [\"torch\", \"torchvision\", \"ftfy\", \"regex\", \"tqdm\"]\n# For compatibility (cannot include special characters in function name)\nmodel_functions = { model: re.sub(f'[{string.punctuation}]', '_', model) for model in _available_models()}\ndef _create_hub_entrypoint(model):\n def entrypoint(**kwargs): \n return _load(model, **kwargs)\n entrypoint.__doc__ = f\"\"\"Loads the {model} CLIP model\n Parameters\n ----------\n device : Union[str, torch.device]\n The device to put the loaded model\n jit : bool\n Whether to load the optimized JIT model or more hackable non-JIT model (default).\n download_root: str\n path to download the model files; by default, it uses \"~/.cache/clip\"\n Returns\n -------\n model : torch.nn.Module\n The {model} CLIP model\n preprocess : Callable[[PIL.Image], torch.Tensor]",
"type": "code",
"location": "/hubconf.py:1-32"
},
"19": {
"file_id": 1,
"content": "This code defines a function _create_hub_entrypoint that creates an entry point for loading CLIP models. It also imports necessary dependencies and maps available model names to remove any special characters for compatibility.",
"type": "comment"
},
"20": {
"file_id": 1,
"content": " A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input\n \"\"\"\n return entrypoint\ndef tokenize():\n return _tokenize\n_entrypoints = {model_functions[model]: _create_hub_entrypoint(model) for model in _available_models()}\nglobals().update(_entrypoints)",
"type": "code",
"location": "/hubconf.py:33-42"
},
"21": {
"file_id": 1,
"content": "This code defines a function that converts a PIL image into a tensor. It also creates entrypoints for different models using _available_models() and updates the global namespace with these entrypoints.",
"type": "comment"
},
"22": {
"file_id": 2,
"content": "/model-card.md",
"type": "filepath"
},
"23": {
"file_id": 2,
"content": "OpenAI's CLIP model is a multimodal AI for computer vision and zero-shot image classification. It uses ResNet50 or Vision Transformer as encoders but has limitations like dataset building, performance variability, and potential biases. Training data includes website crawling and YFCC100M datasets. The code provides a Google Form link for feedback on model performance and risks.",
"type": "summary"
},
"24": {
"file_id": 2,
"content": "# Model Card: CLIP\nInspired by [Model Cards for Model Reporting (Mitchell et al.)](https://arxiv.org/abs/1810.03993) and [Lessons from Archives (Jo & Gebru)](https://arxiv.org/pdf/1912.10389.pdf), we’re providing some accompanying information about the multimodal model.\n## Model Details\nThe CLIP model was developed by researchers at OpenAI to learn about what contributes to robustness in computer vision tasks. The model was also developed to test the ability of models to generalize to arbitrary image classification tasks in a zero-shot manner. It was not developed for general model deployment - to deploy models like CLIP, researchers will first need to carefully study their capabilities in relation to the specific context they’re being deployed within.\n### Model Date\nJanuary 2021\n### Model Type\nThe base model uses a ResNet50 with several modifications as an image encoder and uses a masked self-attention Transformer as a text encoder. These encoders are trained to maximize the similarity of (i",
"type": "code",
"location": "/model-card.md:1-15"
},
"25": {
"file_id": 2,
"content": "Storage location: \"model-card.md\":0-14\nCode description: This code is a model card for CLIP, a multimodal model developed by OpenAI researchers. The model aims to understand what contributes to robustness in computer vision tasks and test generalization abilities in zero-shot image classification tasks. It was not designed for general deployment and requires careful study before being used in specific contexts. The model card provides details on the development date, model type (ResNet50 with modifications as an image encoder and a masked self-attention Transformer as a text encoder), and that the encoders are trained to maximize similarity of inputs.",
"type": "comment"
},
"26": {
"file_id": 2,
"content": "mage, text) pairs via a contrastive loss. There is also a variant of the model where the ResNet image encoder is replaced with a Vision Transformer.\n### Model Versions\nInitially, we’ve released one CLIP model based on the Vision Transformer architecture equivalent to ViT-B/32, along with the RN50 model, using the architecture equivalent to ResNet-50.\nAs part of the staged release process, we have also released the RN101 model, as well as RN50x4, a RN50 scaled up 4x according to the [EfficientNet](https://arxiv.org/abs/1905.11946) scaling rule. In July 2021, we additionally released the RN50x16 and ViT-B/16 models, and in January 2022, the RN50x64 and ViT-L/14 models were released. Lastly, the ViT-L/14@336px model was released in April 2022.\nPlease see the paper linked below for further details about their specification.\n### Documents\n- [Blog Post](https://openai.com/blog/clip/)\n- [CLIP Paper](https://arxiv.org/abs/2103.00020)\n## Model Use\n### Intended Use\nThe model is intended as a research outp",
"type": "code",
"location": "/model-card.md:15-36"
},
"27": {
"file_id": 2,
"content": "This code describes the CLIP model, a contrastive image-text model with variants using Vision Transformer or ResNet image encoder. It mentions the different released versions of the model and provides links to relevant documents such as the blog post and paper for further details on specifications and intended use.",
"type": "comment"
},
"28": {
"file_id": 2,
"content": "ut for research communities. We hope that this model will enable researchers to better understand and explore zero-shot, arbitrary image classification. We also hope it can be used for interdisciplinary studies of the potential impact of such models - the CLIP paper includes a discussion of potential downstream impacts to provide an example for this sort of analysis.\n#### Primary intended uses\nThe primary intended users of these models are AI researchers.\nWe primarily imagine the model will be used by researchers to better understand robustness, generalization, and other capabilities, biases, and constraints of computer vision models.\n### Out-of-Scope Use Cases\n**Any** deployed use case of the model - whether commercial or not - is currently out of scope. Non-deployed use cases such as image search in a constrained environment, are also not recommended unless there is thorough in-domain testing of the model with a specific, fixed class taxonomy. This is because our safety assessment demonst",
"type": "code",
"location": "/model-card.md:36-46"
},
"29": {
"file_id": 2,
"content": "This code snippet provides information about the intended use and out-of-scope use cases for a specific model. It explains that the primary audience is AI researchers who will use it to study various aspects of computer vision models, such as robustness, generalization, capabilities, biases, and constraints. Deployed use cases are currently out of scope, while non-deployed use cases should only be considered after thorough in-domain testing with a fixed class taxonomy.",
"type": "comment"
},
"30": {
"file_id": 2,
"content": "rated a high need for task specific testing especially given the variability of CLIP’s performance with different class taxonomies. This makes untested and unconstrained deployment of the model in any use case currently potentially harmful. \nCertain use cases which would fall under the domain of surveillance and facial recognition are always out-of-scope regardless of performance of the model. This is because the use of artificial intelligence for tasks such as these can be premature currently given the lack of testing norms and checks to ensure its fair use.\nSince the model has not been purposefully trained in or evaluated on any languages other than English, its use should be limited to English language use cases.\n## Data\nThe model was trained on publicly available image-caption data. This was done through a combination of crawling a handful of websites and using commonly-used pre-existing image datasets such as [YFCC100M](http://projects.dfki.uni-kl.de/yfcc100m/). A large portion of the",
"type": "code",
"location": "/model-card.md:46-56"
},
"31": {
"file_id": 2,
"content": "The code highlights the need for task-specific testing due to CLIP's performance variability and cautions against unconstrained deployment in certain use cases. It also emphasizes the model's English language limitations and provides information on the data used for training, including crawling websites and using pre-existing datasets like YFCC100M.",
"type": "comment"
},
"32": {
"file_id": 2,
"content": " data comes from our crawling of the internet. This means that the data is more representative of people and societies most connected to the internet which tend to skew towards more developed nations, and younger, male users.\n### Data Mission Statement\nOur goal with building this dataset was to test out robustness and generalizability in computer vision tasks. As a result, the focus was on gathering large quantities of data from different publicly-available internet data sources. The data was gathered in a mostly non-interventionist manner. However, we only crawled websites that had policies against excessively violent and adult images and allowed us to filter out such content. We do not intend for this dataset to be used as the basis for any commercial or deployed model and will not be releasing the dataset.\n## Performance and Limitations\n### Performance\nWe have evaluated the performance of CLIP on a wide range of benchmarks across a variety of computer vision datasets such as OCR to textu",
"type": "code",
"location": "/model-card.md:56-68"
},
"33": {
"file_id": 2,
"content": "The code is describing the data used in building a dataset, its mission statement, and discussing performance and limitations. The data comes from internet crawling, mainly focusing on more developed nations and younger male users. The goal was to test robustness and generalizability in computer vision tasks. The dataset will not be released for commercial or deployed use. Performance is evaluated across various benchmarks and computer vision datasets like OCR to text.",
"type": "comment"
},
"34": {
"file_id": 2,
"content": "re recognition to fine-grained classification. The paper describes model performance on the following datasets:\n- Food101\n- CIFAR10 \n- CIFAR100 \n- Birdsnap\n- SUN397\n- Stanford Cars\n- FGVC Aircraft\n- VOC2007\n- DTD\n- Oxford-IIIT Pet dataset\n- Caltech101\n- Flowers102\n- MNIST \n- SVHN \n- IIIT5K \n- Hateful Memes \n- SST-2\n- UCF101\n- Kinetics700\n- Country211\n- CLEVR Counting\n- KITTI Distance\n- STL-10\n- RareAct\n- Flickr30\n- MSCOCO\n- ImageNet\n- ImageNet-A\n- ImageNet-R\n- ImageNet Sketch\n- ObjectNet (ImageNet Overlap)\n- Youtube-BB\n- ImageNet-Vid\n## Limitations\nCLIP and our analysis of it have a number of limitations. CLIP currently struggles with respect to certain tasks such as fine grained classification and counting objects. CLIP also poses issues with regards to fairness and bias which we discuss in the paper and briefly in the next section. Additionally, our approach to testing CLIP also has an important limitation- in many cases we have used linear probes to evaluate the performance of CLIP and there is evidence suggesting that linear probes can underestimate model performance.",
"type": "code",
"location": "/model-card.md:68-106"
},
"35": {
"file_id": 2,
"content": "The code lists various datasets used in the evaluation of the model's performance.\n\nIt highlights that CLIP has limitations, such as difficulties with fine-grained classification and counting objects. It also addresses issues related to fairness and bias, while noting a limitation in their approach by using linear probes for evaluation, which may underestimate model performance.",
"type": "comment"
},
"36": {
"file_id": 2,
"content": "### Bias and Fairness\nWe find that the performance of CLIP - and the specific biases it exhibits - can depend significantly on class design and the choices one makes for categories to include and exclude. We tested the risk of certain kinds of denigration with CLIP by classifying images of people from [Fairface](https://arxiv.org/abs/1908.04913) into crime-related and non-human animal categories. We found significant disparities with respect to race and gender. Additionally, we found that these disparities could shift based on how the classes were constructed. (Details captured in the Broader Impacts Section in the paper).\nWe also tested the performance of CLIP on gender, race and age classification using the Fairface dataset (We default to using race categories as they are constructed in the Fairface dataset.) in order to assess quality of performance across different demographics. We found accuracy >96% across all races for gender classification with ‘Middle Eastern’ having the highest",
"type": "code",
"location": "/model-card.md:108-112"
},
"37": {
"file_id": 2,
"content": "Discusses the impact of class design on CLIP's biases, highlights disparities based on race and gender using Fairface dataset, and mentions accuracy over 96% for gender classification across all races.",
"type": "comment"
},
"38": {
"file_id": 2,
"content": " accuracy (98.4%) and ‘White’ having the lowest (96.5%). Additionally, CLIP averaged ~93% for racial classification and ~63% for age classification. Our use of evaluations to test for gender, race and age classification as well as denigration harms is simply to evaluate performance of the model across people and surface potential risks and not to demonstrate an endorsement/enthusiasm for such tasks.\n## Feedback\n### Where to send questions or comments about the model\nPlease use [this Google Form](https://forms.gle/Uv7afRH5dvY34ZEs9)",
"type": "code",
"location": "/model-card.md:112-120"
},
"39": {
"file_id": 2,
"content": "This code is providing the accuracy of the model for various classifications and emphasizing that these evaluations are to test performance and identify potential risks, not to endorse such tasks. It also provides a link to a Google Form for questions or comments about the model.",
"type": "comment"
},
"40": {
"file_id": 3,
"content": "/requirements.txt",
"type": "filepath"
},
"41": {
"file_id": 3,
"content": "Installing required packages: ftfy, regex, tqdm, torch, and torchvision.",
"type": "summary"
},
"42": {
"file_id": 3,
"content": "ftfy\nregex\ntqdm\ntorch\ntorchvision",
"type": "code",
"location": "/requirements.txt:1-5"
},
"43": {
"file_id": 3,
"content": "Installing required packages: ftfy, regex, tqdm, torch, and torchvision.",
"type": "comment"
},
"44": {
"file_id": 4,
"content": "/setup.py",
"type": "filepath"
},
"45": {
"file_id": 4,
"content": "This code sets up a Python package named \"clip\" using setuptools. It imports necessary modules, defines package attributes and requirements, and specifies installation dependencies. It also includes the \"dev\" extra requirement for developers.",
"type": "summary"
},
"46": {
"file_id": 4,
"content": "import os\nimport pkg_resources\nfrom setuptools import setup, find_packages\nsetup(\n name=\"clip\",\n py_modules=[\"clip\"],\n version=\"1.0\",\n description=\"\",\n author=\"OpenAI\",\n packages=find_packages(exclude=[\"tests*\"]),\n install_requires=[\n str(r)\n for r in pkg_resources.parse_requirements(\n open(os.path.join(os.path.dirname(__file__), \"requirements.txt\"))\n )\n ],\n include_package_data=True,\n extras_require={'dev': ['pytest']},\n)",
"type": "code",
"location": "/setup.py:1-21"
},
"47": {
"file_id": 4,
"content": "This code sets up a Python package named \"clip\" using setuptools. It imports necessary modules, defines package attributes and requirements, and specifies installation dependencies. It also includes the \"dev\" extra requirement for developers.",
"type": "comment"
},
"48": {
"file_id": 5,
"content": "/clip/__init__.py",
"type": "filepath"
},
"49": {
"file_id": 5,
"content": "Imports all functions and classes from the \"clip\" module.",
"type": "summary"
},
"50": {
"file_id": 5,
"content": "from .clip import *",
"type": "code",
"location": "/clip/__init__.py:1-1"
},
"51": {
"file_id": 5,
"content": "Imports all functions and classes from the \"clip\" module.",
"type": "comment"
},
"52": {
"file_id": 6,
"content": "/clip/clip.py",
"type": "filepath"
},
"53": {
"file_id": 6,
"content": "The code downloads and handles pre-trained models, including CLIP, checks availability, loads optimized versions, and implements device-specific patches while tokenizing input strings. This function encodes a list of texts using a tokenizer, adds start/end tokens, returns tensor, and optionally truncates if exceeding context length.",
"type": "summary"
},
"54": {
"file_id": 6,
"content": "import hashlib\nimport os\nimport urllib\nimport warnings\nfrom typing import Any, Union, List\nfrom pkg_resources import packaging\nimport torch\nfrom PIL import Image\nfrom torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize\nfrom tqdm import tqdm\nfrom .model import build_model\nfrom .simple_tokenizer import SimpleTokenizer as _Tokenizer\ntry:\n from torchvision.transforms import InterpolationMode\n BICUBIC = InterpolationMode.BICUBIC\nexcept ImportError:\n BICUBIC = Image.BICUBIC\nif packaging.version.parse(torch.__version__) < packaging.version.parse(\"1.7.1\"):\n warnings.warn(\"PyTorch version 1.7.1 or higher is recommended\")\n__all__ = [\"available_models\", \"load\", \"tokenize\"]\n_tokenizer = _Tokenizer()\n_MODELS = {\n \"RN50\": \"https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt\",\n \"RN101\": \"https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt\",\n ",
"type": "code",
"location": "/clip/clip.py:1-33"
},
"55": {
"file_id": 6,
"content": "Importing necessary libraries and defining variables for model choices and tokenizer.",
"type": "comment"
},
"56": {
"file_id": 6,
"content": "\"RN50x4\": \"https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt\",\n \"RN50x16\": \"https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt\",\n \"RN50x64\": \"https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt\",\n \"ViT-B/32\": \"https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt\",\n \"ViT-B/16\": \"https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt\",\n \"ViT-L/14\": \"https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt\",\n \"ViT-L/14@336px\": \"https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt\",\n}\ndef _download(url: str, root: str):",
"type": "code",
"location": "/clip/clip.py:33-43"
},
"57": {
"file_id": 6,
"content": "This code defines a dictionary of URLs for different pre-trained models and includes a function _download() to download the model files.",
"type": "comment"
},
"58": {
"file_id": 6,
"content": " os.makedirs(root, exist_ok=True)\n filename = os.path.basename(url)\n expected_sha256 = url.split(\"/\")[-2]\n download_target = os.path.join(root, filename)\n if os.path.exists(download_target) and not os.path.isfile(download_target):\n raise RuntimeError(f\"{download_target} exists and is not a regular file\")\n if os.path.isfile(download_target):\n if hashlib.sha256(open(download_target, \"rb\").read()).hexdigest() == expected_sha256:\n return download_target\n else:\n warnings.warn(f\"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file\")\n with urllib.request.urlopen(url) as source, open(download_target, \"wb\") as output:\n with tqdm(total=int(source.info().get(\"Content-Length\")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop:\n while True:\n buffer = source.read(8192)\n if not buffer:\n break\n output.write(buffer)\n loop.update(len(buffer))",
"type": "code",
"location": "/clip/clip.py:44-67"
},
"59": {
"file_id": 6,
"content": "Creates a directory and checks if the file already exists, then downloads or verifies the file's SHA256 checksum.",
"type": "comment"
},
"60": {
"file_id": 6,
"content": " if hashlib.sha256(open(download_target, \"rb\").read()).hexdigest() != expected_sha256:\n raise RuntimeError(\"Model has been downloaded but the SHA256 checksum does not not match\")\n return download_target\ndef _convert_image_to_rgb(image):\n return image.convert(\"RGB\")\ndef _transform(n_px):\n return Compose([\n Resize(n_px, interpolation=BICUBIC),\n CenterCrop(n_px),\n _convert_image_to_rgb,\n ToTensor(),\n Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),\n ])\ndef available_models() -> List[str]:\n \"\"\"Returns the names of available CLIP models\"\"\"\n return list(_MODELS.keys())\ndef load(name: str, device: Union[str, torch.device] = \"cuda\" if torch.cuda.is_available() else \"cpu\", jit: bool = False, download_root: str = None):\n \"\"\"Load a CLIP model\n Parameters\n ----------\n name : str\n A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict\n device : Union[str, torch.device]",
"type": "code",
"location": "/clip/clip.py:69-102"
},
"61": {
"file_id": 6,
"content": "Code checks the SHA256 checksum of a downloaded model and raises an error if it doesn't match. It also defines functions for image transformation, loading available CLIP models, and converting images to RGB format.",
"type": "comment"
},
"62": {
"file_id": 6,
"content": " The device to put the loaded model\n jit : bool\n Whether to load the optimized JIT model or more hackable non-JIT model (default).\n download_root: str\n path to download the model files; by default, it uses \"~/.cache/clip\"\n Returns\n -------\n model : torch.nn.Module\n The CLIP model\n preprocess : Callable[[PIL.Image], torch.Tensor]\n A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input\n \"\"\"\n if name in _MODELS:\n model_path = _download(_MODELS[name], download_root or os.path.expanduser(\"~/.cache/clip\"))\n elif os.path.isfile(name):\n model_path = name\n else:\n raise RuntimeError(f\"Model {name} not found; available models = {available_models()}\")\n with open(model_path, 'rb') as opened_file:\n try:\n # loading JIT archive\n model = torch.jit.load(opened_file, map_location=device if jit else \"cpu\").eval()\n state_dict = None\n except RuntimeError:",
"type": "code",
"location": "/clip/clip.py:103-131"
},
"63": {
"file_id": 6,
"content": "The code downloads the CLIP model based on the specified name and device. It checks if the model is available as a file or downloads it from the provided root path. The JIT-optimized version of the model is loaded if 'jit' is True, otherwise, the non-JIT version is used.",
"type": "comment"
},
"64": {
"file_id": 6,
"content": " # loading saved state dict\n if jit:\n warnings.warn(f\"File {model_path} is not a JIT archive. Loading as a state dict instead\")\n jit = False\n state_dict = torch.load(opened_file, map_location=\"cpu\")\n if not jit:\n model = build_model(state_dict or model.state_dict()).to(device)\n if str(device) == \"cpu\":\n model.float()\n return model, _transform(model.visual.input_resolution)\n # patch the device names\n device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])\n device_node = [n for n in device_holder.graph.findAllNodes(\"prim::Constant\") if \"Device\" in repr(n)][-1]\n def _node_get(node: torch._C.Node, key: str):\n \"\"\"Gets attributes of a node which is polymorphic over return type.\n From https://github.com/pytorch/pytorch/pull/82628\n \"\"\"\n sel = node.kindOf(key)\n return getattr(node, sel)(key)\n def patch_device(module):\n try:",
"type": "code",
"location": "/clip/clip.py:132-157"
},
"65": {
"file_id": 6,
"content": "Loading saved state dict and handling JIT (Just-In-Time) support.\nIf not a JIT archive, loading as a state dict instead.\nLoading model with or without JIT support depending on jit variable.\nConverting model to float if device is CPU.\nReturning the model and transformed input resolution.\nPatching device names using torch.jit.trace.",
"type": "comment"
},
"66": {
"file_id": 6,
"content": " graphs = [module.graph] if hasattr(module, \"graph\") else []\n except RuntimeError:\n graphs = []\n if hasattr(module, \"forward1\"):\n graphs.append(module.forward1.graph)\n for graph in graphs:\n for node in graph.findAllNodes(\"prim::Constant\"):\n if \"value\" in node.attributeNames() and str(_node_get(node, \"value\")).startswith(\"cuda\"):\n node.copyAttributes(device_node)\n model.apply(patch_device)\n patch_device(model.encode_image)\n patch_device(model.encode_text)\n # patch dtype to float32 on CPU\n if str(device) == \"cpu\":\n float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])\n float_input = list(float_holder.graph.findNode(\"aten::to\").inputs())[1]\n float_node = float_input.node()\n def patch_float(module):\n try:\n graphs = [module.graph] if hasattr(module, \"graph\") else []\n except RuntimeError:\n graphs = []",
"type": "code",
"location": "/clip/clip.py:158-184"
},
"67": {
"file_id": 6,
"content": "Applying device-specific patches to the model's graph.",
"type": "comment"
},
"68": {
"file_id": 6,
"content": " if hasattr(module, \"forward1\"):\n graphs.append(module.forward1.graph)\n for graph in graphs:\n for node in graph.findAllNodes(\"aten::to\"):\n inputs = list(node.inputs())\n for i in [1, 2]: # dtype can be the second or third argument to aten::to()\n if _node_get(inputs[i].node(), \"value\") == 5:\n inputs[i].node().copyAttributes(float_node)\n model.apply(patch_float)\n patch_float(model.encode_image)\n patch_float(model.encode_text)\n model.float()\n return model, _transform(model.input_resolution.item())\ndef tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]:\n \"\"\"\n Returns the tokenized representation of given input string(s)\n Parameters\n ----------\n texts : Union[str, List[str]]\n An input string or a list of input strings to tokenize\n context_length : int",
"type": "code",
"location": "/clip/clip.py:186-214"
},
"69": {
"file_id": 6,
"content": "The code is applying a patch to select functions in the model to convert certain types to floats, then tokenizes input strings based on context_length.",
"type": "comment"
},
"70": {
"file_id": 6,
"content": " The context length to use; all CLIP models use 77 as the context length\n truncate: bool\n Whether to truncate the text in case its encoding is longer than the context length\n Returns\n -------\n A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length].\n We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long.\n \"\"\"\n if isinstance(texts, str):\n texts = [texts]\n sot_token = _tokenizer.encoder[\"<|startoftext|>\"]\n eot_token = _tokenizer.encoder[\"<|endoftext|>\"]\n all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]\n if packaging.version.parse(torch.__version__) < packaging.version.parse(\"1.8.0\"):\n result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)\n else:\n result = torch.zeros(len(all_tokens), context_length, dtype=torch.int)\n for i, tokens in enumerate(all_tokens):\n if len(tokens) > context_length:",
"type": "code",
"location": "/clip/clip.py:215-237"
},
"71": {
"file_id": 6,
"content": "This function takes a list of texts and encodes them using the tokenizer. It then adds start and end of text tokens, and returns a tensor of shape [number of input strings, context_length]. If torch version is <1.8.0, it uses LongTensor for indices.",
"type": "comment"
},
"72": {
"file_id": 6,
"content": " if truncate:\n tokens = tokens[:context_length]\n tokens[-1] = eot_token\n else:\n raise RuntimeError(f\"Input {texts[i]} is too long for context length {context_length}\")\n result[i, :len(tokens)] = torch.tensor(tokens)\n return result",
"type": "code",
"location": "/clip/clip.py:238-245"
},
"73": {
"file_id": 6,
"content": "If truncate is True, only keep the first context_length tokens and set the last token to eot_token. If not, raise an error if input text exceeds the context length. Store the tokens as a torch tensor in result.",
"type": "comment"
},
"74": {
"file_id": 7,
"content": "/clip/model.py",
"type": "filepath"
},
"75": {
"file_id": 7,
"content": "A summary of the comments discusses implementing advanced models with deep learning and attention mechanisms, using CLIP models for Convolutional Neural Networks and VisionTransformers, and initializing, converting, and loading state dicts into a CLIP model for evaluation.",
"type": "summary"
},
"76": {
"file_id": 7,
"content": "from collections import OrderedDict\nfrom typing import Tuple, Union\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nclass Bottleneck(nn.Module):\n expansion = 4\n def __init__(self, inplanes, planes, stride=1):\n super().__init__()\n # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1\n self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)\n self.bn1 = nn.BatchNorm2d(planes)\n self.relu1 = nn.ReLU(inplace=True)\n self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(planes)\n self.relu2 = nn.ReLU(inplace=True)\n self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()\n self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)\n self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n self.relu3 = nn.ReLU(inplace=True)\n self.downsample = None\n self.stride = stride\n if stride > 1 or inplanes != planes * Bottleneck.expansion:",
"type": "code",
"location": "/clip/model.py:1-34"
},
"77": {
"file_id": 7,
"content": "Class Bottleneck is defined as a subclass of nn.Module for residual block in a Convolutional Neural Network (CNN) architecture. It performs multiple convolutions, batch normalization, and activation functions. If stride > 1, it also includes an average pooling layer. The downsample parameter is set to None here but can be used if input and output planes are different.",
"type": "comment"
},
"78": {
"file_id": 7,
"content": " # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1\n self.downsample = nn.Sequential(OrderedDict([\n (\"-1\", nn.AvgPool2d(stride)),\n (\"0\", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),\n (\"1\", nn.BatchNorm2d(planes * self.expansion))\n ]))\n def forward(self, x: torch.Tensor):\n identity = x\n out = self.relu1(self.bn1(self.conv1(x)))\n out = self.relu2(self.bn2(self.conv2(out)))\n out = self.avgpool(out)\n out = self.bn3(self.conv3(out))\n if self.downsample is not None:\n identity = self.downsample(x)\n out += identity\n out = self.relu3(out)\n return out\nclass AttentionPool2d(nn.Module):\n def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):\n super().__init__()\n self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)",
"type": "code",
"location": "/clip/model.py:35-61"
},
"79": {
"file_id": 7,
"content": "This code defines a convolutional block with downsampling and an AttentionPool2d module. The convolutional block performs convolutions with batch normalization and ReLU activations, while also allowing for optional downsampling through the defined `downsample` layer. The AttentionPool2d module is responsible for processing spatial features of input data using attention mechanism.",
"type": "comment"
},
"80": {
"file_id": 7,
"content": " self.k_proj = nn.Linear(embed_dim, embed_dim)\n self.q_proj = nn.Linear(embed_dim, embed_dim)\n self.v_proj = nn.Linear(embed_dim, embed_dim)\n self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)\n self.num_heads = num_heads\n def forward(self, x):\n x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC\n x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC\n x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC\n x, _ = F.multi_head_attention_forward(\n query=x[:1], key=x, value=x,\n embed_dim_to_check=x.shape[-1],\n num_heads=self.num_heads,\n q_proj_weight=self.q_proj.weight,\n k_proj_weight=self.k_proj.weight,\n v_proj_weight=self.v_proj.weight,\n in_proj_weight=None,\n in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),\n bias_k=None,\n bias_v=None,\n add_zero_attn=False,",
"type": "code",
"location": "/clip/model.py:62-83"
},
"81": {
"file_id": 7,
"content": "The code defines a class with `forward` method and initializes the necessary linear layers (`k_proj`, `q_proj`, `v_proj`, `c_proj`) for multi-head attention. It then processes input `x` by flattening, concatenating, adding positional embeddings, and finally calling `F.multi_head_attention_forward` with appropriate arguments.",
"type": "comment"
},
"82": {
"file_id": 7,
"content": " dropout_p=0,\n out_proj_weight=self.c_proj.weight,\n out_proj_bias=self.c_proj.bias,\n use_separate_proj_weight=True,\n training=self.training,\n need_weights=False\n )\n return x.squeeze(0)\nclass ModifiedResNet(nn.Module):\n \"\"\"\n A ResNet class that is similar to torchvision's but contains the following changes:\n - There are now 3 \"stem\" convolutions as opposed to 1, with an average pool instead of a max pool.\n - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1\n - The final pooling layer is a QKV attention instead of an average pool\n \"\"\"\n def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):\n super().__init__()\n self.output_dim = output_dim\n self.input_resolution = input_resolution\n # the 3-layer stem\n self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)\n self.bn1 = nn.BatchNorm2d(width // 2)",
"type": "code",
"location": "/clip/model.py:84-109"
},
"83": {
"file_id": 7,
"content": "Code snippet initializes a Conv2d layer, followed by BatchNorm2d layer for the stem of the modified ResNet. The stem consists of 3 convolution layers, each with stride 2 and padding 1. The BatchNorm2d layer normalizes the output of the convolution layer.",
"type": "comment"
},
"84": {
"file_id": 7,
"content": " self.relu1 = nn.ReLU(inplace=True)\n self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(width // 2)\n self.relu2 = nn.ReLU(inplace=True)\n self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)\n self.bn3 = nn.BatchNorm2d(width)\n self.relu3 = nn.ReLU(inplace=True)\n self.avgpool = nn.AvgPool2d(2)\n # residual layers\n self._inplanes = width # this is a *mutable* variable used during construction\n self.layer1 = self._make_layer(width, layers[0])\n self.layer2 = self._make_layer(width * 2, layers[1], stride=2)\n self.layer3 = self._make_layer(width * 4, layers[2], stride=2)\n self.layer4 = self._make_layer(width * 8, layers[3], stride=2)\n embed_dim = width * 32 # the ResNet feature dimension\n self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)\n def _make_layer(self, planes, blocks, stride=1):",
"type": "code",
"location": "/clip/model.py:110-129"
},
"85": {
"file_id": 7,
"content": "Code is defining a ResNet model with various layers such as convolution, batch normalization, ReLU activation, average pooling, and residual layers. It also includes an attention pooling layer. The ResNet model's feature dimension is set to 32.",
"type": "comment"
},
"86": {
"file_id": 7,
"content": " layers = [Bottleneck(self._inplanes, planes, stride)]\n self._inplanes = planes * Bottleneck.expansion\n for _ in range(1, blocks):\n layers.append(Bottleneck(self._inplanes, planes))\n return nn.Sequential(*layers)\n def forward(self, x):\n def stem(x):\n x = self.relu1(self.bn1(self.conv1(x)))\n x = self.relu2(self.bn2(self.conv2(x)))\n x = self.relu3(self.bn3(self.conv3(x)))\n x = self.avgpool(x)\n return x\n x = x.type(self.conv1.weight.dtype)\n x = stem(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n x = self.attnpool(x)\n return x\nclass LayerNorm(nn.LayerNorm):\n \"\"\"Subclass torch's LayerNorm to handle fp16.\"\"\"\n def forward(self, x: torch.Tensor):\n orig_type = x.dtype\n ret = super().forward(x.type(torch.float32))\n return ret.type(orig_type)\nclass QuickGELU(nn.Module):\n def forward(self, x: torch.Tensor):",
"type": "code",
"location": "/clip/model.py:130-167"
},
"87": {
"file_id": 7,
"content": "129-138: Initialize layers with a Bottleneck block.\n140-146: Update inplanes for subsequent blocks.\n147-152: Append additional Bottleneck blocks to layers list.\n153-159: Return a nn.Sequential model from the layers list.\n160-166: Implement forward pass of the model, including stem and layer blocks.",
"type": "comment"
},
"88": {
"file_id": 7,
"content": " return x * torch.sigmoid(1.702 * x)\nclass ResidualAttentionBlock(nn.Module):\n def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None):\n super().__init__()\n self.attn = nn.MultiheadAttention(d_model, n_head)\n self.ln_1 = LayerNorm(d_model)\n self.mlp = nn.Sequential(OrderedDict([\n (\"c_fc\", nn.Linear(d_model, d_model * 4)),\n (\"gelu\", QuickGELU()),\n (\"c_proj\", nn.Linear(d_model * 4, d_model))\n ]))\n self.ln_2 = LayerNorm(d_model)\n self.attn_mask = attn_mask\n def attention(self, x: torch.Tensor):\n self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None\n return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]\n def forward(self, x: torch.Tensor):\n x = x + self.attention(self.ln_1(x))\n x = x + self.mlp(self.ln_2(x))\n return x\nclass Transformer(nn.Module):\n def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None):",
"type": "code",
"location": "/clip/model.py:168-196"
},
"89": {
"file_id": 7,
"content": "This code defines a Transformer model, specifically the Residual Attention Block and the main Transformer class. The ResidualAttentionBlock contains a MultiheadAttention layer, LayerNorm layers, and a feed-forward network. The Transformer class is initialized with width (d_model), number of layers, and number of heads for attention mechanism. It also accepts an optional attn_mask tensor.",
"type": "comment"
},
"90": {
"file_id": 7,
"content": " super().__init__()\n self.width = width\n self.layers = layers\n self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)])\n def forward(self, x: torch.Tensor):\n return self.resblocks(x)\nclass VisionTransformer(nn.Module):\n def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int):\n super().__init__()\n self.input_resolution = input_resolution\n self.output_dim = output_dim\n self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)\n scale = width ** -0.5\n self.class_embedding = nn.Parameter(scale * torch.randn(width))\n self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))\n self.ln_pre = LayerNorm(width)\n self.transformer = Transformer(width, layers, heads)\n self.ln_post = LayerNorm(width)",
"type": "code",
"location": "/clip/model.py:197-220"
},
"91": {
"file_id": 7,
"content": "This code defines a VisionTransformer model with an input resolution, patch size, width, layers, number of heads, and output dimension. It initializes the model's parameters and contains forward pass and transformer class definitions.",
"type": "comment"
},
"92": {
"file_id": 7,
"content": " self.proj = nn.Parameter(scale * torch.randn(width, output_dim))\n def forward(self, x: torch.Tensor):\n x = self.conv1(x) # shape = [*, width, grid, grid]\n x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2]\n x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]\n x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width]\n x = x + self.positional_embedding.to(x.dtype)\n x = self.ln_pre(x)\n x = x.permute(1, 0, 2) # NLD -> LND\n x = self.transformer(x)\n x = x.permute(1, 0, 2) # LND -> NLD\n x = self.ln_post(x[:, 0, :])\n if self.proj is not None:\n x = x @ self.proj\n return x\nclass CLIP(nn.Module):\n def __init__(self,\n embed_dim: int,\n # vision\n image_resolution: int,\n vision_layers: Union[Tuple[int, int, int, int], int],",
"type": "code",
"location": "/clip/model.py:221-248"
},
"93": {
"file_id": 7,
"content": "This code defines a CLIP model, which consists of a convolutional layer followed by a Transformer. It performs feature extraction from an input image and then processes the features with a transformer network. The proj parameter is used for applying final linear projection if not None.\nCode location: \"clip/model.py\":249-271",
"type": "comment"
},
"94": {
"file_id": 7,
"content": " vision_width: int,\n vision_patch_size: int,\n # text\n context_length: int,\n vocab_size: int,\n transformer_width: int,\n transformer_heads: int,\n transformer_layers: int\n ):\n super().__init__()\n self.context_length = context_length\n if isinstance(vision_layers, (tuple, list)):\n vision_heads = vision_width * 32 // 64\n self.visual = ModifiedResNet(\n layers=vision_layers,\n output_dim=embed_dim,\n heads=vision_heads,\n input_resolution=image_resolution,\n width=vision_width\n )\n else:\n vision_heads = vision_width // 64\n self.visual = VisionTransformer(\n input_resolution=image_resolution,\n patch_size=vision_patch_size,\n width=vision_width,\n layers=vision_layers,\n heads=vision_heads,",
"type": "code",
"location": "/clip/model.py:249-278"
},
"95": {
"file_id": 7,
"content": "Initializing a model with provided parameters for vision and language processing.",
"type": "comment"
},
"96": {
"file_id": 7,
"content": " output_dim=embed_dim\n )\n self.transformer = Transformer(\n width=transformer_width,\n layers=transformer_layers,\n heads=transformer_heads,\n attn_mask=self.build_attention_mask()\n )\n self.vocab_size = vocab_size\n self.token_embedding = nn.Embedding(vocab_size, transformer_width)\n self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))\n self.ln_final = LayerNorm(transformer_width)\n self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))\n self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))\n self.initialize_parameters()\n def initialize_parameters(self):\n nn.init.normal_(self.token_embedding.weight, std=0.02)\n nn.init.normal_(self.positional_embedding, std=0.01)\n if isinstance(self.visual, ModifiedResNet):\n if self.visual.attnpool is not None:\n std = self.visual.attnpool.c_proj.in_features ** -0.5",
"type": "code",
"location": "/clip/model.py:279-305"
},
"97": {
"file_id": 7,
"content": "This code initializes the model's parameters. It sets up layers such as transformer, token embedding, positional embedding, layer normalization, and logit scale. The initialize_parameters method is used to set up initial values for the embeddings with small standard deviations.",
"type": "comment"
},
"98": {
"file_id": 7,
"content": " nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)\n nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)\n nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)\n nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)\n for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:\n for name, param in resnet_block.named_parameters():\n if name.endswith(\"bn3.weight\"):\n nn.init.zeros_(param)\n proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)\n attn_std = self.transformer.width ** -0.5\n fc_std = (2 * self.transformer.width) ** -0.5\n for block in self.transformer.resblocks:\n nn.init.normal_(block.attn.in_proj_weight, std=attn_std)\n nn.init.normal_(block.attn.out_proj.weight, std=proj_std)\n nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)",
"type": "code",
"location": "/clip/model.py:306-322"
},
"99": {
"file_id": 7,
"content": "This code initializes the weights of various layers in a neural network model. It uses different initialization methods and standards deviations for different types of layers, such as normalizing the weights for attention pools, ResNet blocks, and feedforward layers.",
"type": "comment"
}
}

303
docs/data/1.json Normal file
View File

@ -0,0 +1,303 @@
{
"100": {
"file_id": 7,
"content": " nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)\n if self.text_projection is not None:\n nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)\n def build_attention_mask(self):\n # lazily create causal attention mask, with full attention between the vision tokens\n # pytorch uses additive attention mask; fill with -inf\n mask = torch.empty(self.context_length, self.context_length)\n mask.fill_(float(\"-inf\"))\n mask.triu_(1) # zero out the lower diagonal\n return mask\n @property\n def dtype(self):\n return self.visual.conv1.weight.dtype\n def encode_image(self, image):\n return self.visual(image.type(self.dtype))\n def encode_text(self, text):\n x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model]\n x = x + self.positional_embedding.type(self.dtype)\n x = x.permute(1, 0, 2) # NLD -> LND\n x = self.transformer(x)\n x = x.permute(1, 0, 2) # LND -> NLD",
"type": "code",
"location": "/clip/model.py:323-349"
},
"101": {
"file_id": 7,
"content": "1. Initializes the model parameters with normal distribution.\n2. Builds a causal attention mask for the transformer, filling with -inf for lower diagonal elements.\n3. Encodes image using the provided visual encoder.\n4. Encodes text using token embedding and positional embedding followed by the transformer.\n5. Permutes the output to ensure it's in NLD format (batch_size, sequence_length, feature_dimensions).",
"type": "comment"
},
"102": {
"file_id": 7,
"content": " x = self.ln_final(x).type(self.dtype)\n # x.shape = [batch_size, n_ctx, transformer.width]\n # take features from the eot embedding (eot_token is the highest number in each sequence)\n x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection\n return x\n def forward(self, image, text):\n image_features = self.encode_image(image)\n text_features = self.encode_text(text)\n # normalized features\n image_features = image_features / image_features.norm(dim=1, keepdim=True)\n text_features = text_features / text_features.norm(dim=1, keepdim=True)\n # cosine similarity as logits\n logit_scale = self.logit_scale.exp()\n logits_per_image = logit_scale * image_features @ text_features.t()\n logits_per_text = logits_per_image.t()\n # shape = [global_batch_size, global_batch_size]\n return logits_per_image, logits_per_text\ndef convert_weights(model: nn.Module):\n \"\"\"Convert applicable model parameters to fp16\"\"\"",
"type": "code",
"location": "/clip/model.py:350-376"
},
"103": {
"file_id": 7,
"content": "\"clip/model.py\":349-375",
"type": "comment"
},
"104": {
"file_id": 7,
"content": " def _convert_weights_to_fp16(l):\n if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):\n l.weight.data = l.weight.data.half()\n if l.bias is not None:\n l.bias.data = l.bias.data.half()\n if isinstance(l, nn.MultiheadAttention):\n for attr in [*[f\"{s}_proj_weight\" for s in [\"in\", \"q\", \"k\", \"v\"]], \"in_proj_bias\", \"bias_k\", \"bias_v\"]:\n tensor = getattr(l, attr)\n if tensor is not None:\n tensor.data = tensor.data.half()\n for name in [\"text_projection\", \"proj\"]:\n if hasattr(l, name):\n attr = getattr(l, name)\n if attr is not None:\n attr.data = attr.data.half()\n model.apply(_convert_weights_to_fp16)\ndef build_model(state_dict: dict):\n vit = \"visual.proj\" in state_dict\n if vit:\n vision_width = state_dict[\"visual.conv1.weight\"].shape[0]\n vision_layers = len([k for k in state_dict.keys() if k.startswith(\"visual.\") and k.endswith(\".attn.in_proj_weight\")])",
"type": "code",
"location": "/clip/model.py:378-404"
},
"105": {
"file_id": 7,
"content": "This code applies a function to convert weights of certain layers (Conv1d, Conv2d, Linear, MultiheadAttention) from float32 to float16. It also builds a model using a state dictionary.",
"type": "comment"
},
"106": {
"file_id": 7,
"content": " vision_patch_size = state_dict[\"visual.conv1.weight\"].shape[-1]\n grid_size = round((state_dict[\"visual.positional_embedding\"].shape[0] - 1) ** 0.5)\n image_resolution = vision_patch_size * grid_size\n else:\n counts: list = [len(set(k.split(\".\")[2] for k in state_dict if k.startswith(f\"visual.layer{b}\"))) for b in [1, 2, 3, 4]]\n vision_layers = tuple(counts)\n vision_width = state_dict[\"visual.layer1.0.conv1.weight\"].shape[0]\n output_width = round((state_dict[\"visual.attnpool.positional_embedding\"].shape[0] - 1) ** 0.5)\n vision_patch_size = None\n assert output_width ** 2 + 1 == state_dict[\"visual.attnpool.positional_embedding\"].shape[0]\n image_resolution = output_width * 32\n embed_dim = state_dict[\"text_projection\"].shape[1]\n context_length = state_dict[\"positional_embedding\"].shape[0]\n vocab_size = state_dict[\"token_embedding.weight\"].shape[0]\n transformer_width = state_dict[\"ln_final.weight\"].shape[0]\n transformer_heads = transformer_width // 64",
"type": "code",
"location": "/clip/model.py:405-421"
},
"107": {
"file_id": 7,
"content": "Determine the vision layers' count, widths, and image resolution.\n- Determines how many vision layers exist for each layer number (1, 2, 3, 4) by counting unique keys with matching prefixes in the state dictionary.\n- If the \"visual.attnpool\" key exists, calculates the number of patches along one dimension based on the positional embedding shape and sets vision_patch_size to None. Asserts that the shape matches a specific condition.\n- Computes the image resolution by multiplying vision_patch_size with grid size (rounded down integer value of square root of positional embedding's shape[0] minus one).\n- If no \"visual.attnpool\" key exists, calculates the vision layer count and width based on keys matching prefixes in the state dictionary. Calculates output_width similarly to grid size calculation above but for the attention pooling case.\n- Sets vision_patch_size to None since it's not available from the state dictionary.\n- Finally, calculates image resolution by multiplying output_width with a fixed value (32).\n- Determines embed_dim, context_length and vocab_size based on matching keys in the state dictionary.",
"type": "comment"
},
"108": {
"file_id": 7,
"content": " transformer_layers = len(set(k.split(\".\")[2] for k in state_dict if k.startswith(\"transformer.resblocks\")))\n model = CLIP(\n embed_dim,\n image_resolution, vision_layers, vision_width, vision_patch_size,\n context_length, vocab_size, transformer_width, transformer_heads, transformer_layers\n )\n for key in [\"input_resolution\", \"context_length\", \"vocab_size\"]:\n if key in state_dict:\n del state_dict[key]\n convert_weights(model)\n model.load_state_dict(state_dict)\n return model.eval()",
"type": "code",
"location": "/clip/model.py:422-436"
},
"109": {
"file_id": 7,
"content": "This code initializes a CLIP model with given dimensions and layers, removes unnecessary state dict keys, converts weights, and loads the modified state dict into the model for evaluation.",
"type": "comment"
},
"110": {
"file_id": 8,
"content": "/clip/simple_tokenizer.py",
"type": "filepath"
},
"111": {
"file_id": 8,
"content": "The code defines a `SimpleTokenizer` class that uses Byte Pair Encoding (BPE) for text tokenization, cleans text data, and provides encode and decode functions. It iterates through word characters to form new words by identifying bigrams and breaks when only one character remains.",
"type": "summary"
},
"112": {
"file_id": 8,
"content": "import gzip\nimport html\nimport os\nfrom functools import lru_cache\nimport ftfy\nimport regex as re\n@lru_cache()\ndef default_bpe():\n return os.path.join(os.path.dirname(os.path.abspath(__file__)), \"bpe_simple_vocab_16e6.txt.gz\")\n@lru_cache()\ndef bytes_to_unicode():\n \"\"\"\n Returns list of utf-8 byte and a corresponding list of unicode strings.\n The reversible bpe codes work on unicode strings.\n This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n This is a signficant percentage of your normal, say, 32K bpe vocab.\n To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n And avoids mapping to whitespace/control characters the bpe code barfs on.\n \"\"\"\n bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"¡\"), ord(\"¬\")+1))+list(range(ord(\"®\"), ord(\"ÿ\")+1))\n cs = bs[:]\n n = 0\n for b in range(2**8):\n if b not in bs:",
"type": "code",
"location": "/clip/simple_tokenizer.py:1-30"
},
"113": {
"file_id": 8,
"content": "This code defines two functions: `default_bpe()` and `bytes_to_unicode()`. The `default_bpe()` function returns the path to the \"bpe_simple_vocab_16e6.txt.gz\" file, which seems to be a byte-pair encoding (BPE) vocabulary file. The `bytes_to_unicode()` function creates two lists - one containing Unicode characters from \"!\" to \"~\", and another containing characters from \"¡\" to \"¬\" and \"®\" to \"ÿ\". It then iterates through all 256 possible byte values, checking if they are not in the defined character ranges. If a value is not found in these ranges, it adds it to both lists. The function aims to create lookup tables between utf-8 bytes and Unicode strings for efficient BPE encoding.",
"type": "comment"
},
"114": {
"file_id": 8,
"content": " bs.append(b)\n cs.append(2**8+n)\n n += 1\n cs = [chr(n) for n in cs]\n return dict(zip(bs, cs))\ndef get_pairs(word):\n \"\"\"Return set of symbol pairs in a word.\n Word is represented as tuple of symbols (symbols being variable-length strings).\n \"\"\"\n pairs = set()\n prev_char = word[0]\n for char in word[1:]:\n pairs.add((prev_char, char))\n prev_char = char\n return pairs\ndef basic_clean(text):\n text = ftfy.fix_text(text)\n text = html.unescape(html.unescape(text))\n return text.strip()\ndef whitespace_clean(text):\n text = re.sub(r'\\s+', ' ', text)\n text = text.strip()\n return text\nclass SimpleTokenizer(object):\n def __init__(self, bpe_path: str = default_bpe()):\n self.byte_encoder = bytes_to_unicode()\n self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}\n merges = gzip.open(bpe_path).read().decode(\"utf-8\").split('\\n')\n merges = merges[1:49152-256-2+1]\n merges = [tuple(merge.split()) for merge in merges]",
"type": "code",
"location": "/clip/simple_tokenizer.py:31-68"
},
"115": {
"file_id": 8,
"content": "This code defines a class `SimpleTokenizer` that performs text tokenization using Byte Pair Encoding (BPE). It also includes functions for cleaning text data, such as removing special characters, fixing text, and handling whitespace.",
"type": "comment"
},
"116": {
"file_id": 8,
"content": " vocab = list(bytes_to_unicode().values())\n vocab = vocab + [v+'</w>' for v in vocab]\n for merge in merges:\n vocab.append(''.join(merge))\n vocab.extend(['<|startoftext|>', '<|endoftext|>'])\n self.encoder = dict(zip(vocab, range(len(vocab))))\n self.decoder = {v: k for k, v in self.encoder.items()}\n self.bpe_ranks = dict(zip(merges, range(len(merges))))\n self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}\n self.pat = re.compile(r\"\"\"<\\|startoftext\\|>|<\\|endoftext\\|>|'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+\"\"\", re.IGNORECASE)\n def bpe(self, token):\n if token in self.cache:\n return self.cache[token]\n word = tuple(token[:-1]) + ( token[-1] + '</w>',)\n pairs = get_pairs(word)\n if not pairs:\n return token+'</w>'\n while True:\n bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))\n if bigram not in self.bpe_ranks:",
"type": "code",
"location": "/clip/simple_tokenizer.py:69-91"
},
"117": {
"file_id": 8,
"content": "This code defines a class for tokenizing text using Byte Pair Encoding (BPE). It creates the vocabulary, encoder and decoder dictionaries, initializes BPE ranks and cache. The bpe method takes a token, checks if it's in the cache, and if not, applies BPE until it reaches a single character or an existing BPE word.",
"type": "comment"
},
"118": {
"file_id": 8,
"content": " break\n first, second = bigram\n new_word = []\n i = 0\n while i < len(word):\n try:\n j = word.index(first, i)\n new_word.extend(word[i:j])\n i = j\n except:\n new_word.extend(word[i:])\n break\n if word[i] == first and i < len(word)-1 and word[i+1] == second:\n new_word.append(first+second)\n i += 2\n else:\n new_word.append(word[i])\n i += 1\n new_word = tuple(new_word)\n word = new_word\n if len(word) == 1:\n break\n else:\n pairs = get_pairs(word)\n word = ' '.join(word)\n self.cache[token] = word\n return word\n def encode(self, text):\n bpe_tokens = []\n text = whitespace_clean(basic_clean(text)).lower()\n for token in re.findall(self.pat, text):",
"type": "code",
"location": "/clip/simple_tokenizer.py:92-124"
},
"119": {
"file_id": 8,
"content": "Iterates through word characters and forms a new word by identifying bigrams (pairs of consecutive characters), joining single characters, and breaking when only one character remains. Stores the result in self.cache after converting it to a string.",
"type": "comment"
},
"120": {
"file_id": 8,
"content": " token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))\n bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))\n return bpe_tokens\n def decode(self, tokens):\n text = ''.join([self.decoder[token] for token in tokens])\n text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=\"replace\").replace('</w>', ' ')\n return text",
"type": "code",
"location": "/clip/simple_tokenizer.py:125-132"
},
"121": {
"file_id": 8,
"content": "This code defines a class for tokenization using byte encoding, BPE (Byte Pair Encoding), and provides decode function. It encodes a token into its byte representation, splits it with BPE, and stores the resulting tokens in bpe_tokens list. The decode method reconstructs the original text from the token list using the decoder mapping.",
"type": "comment"
},
"122": {
"file_id": 9,
"content": "/data/country211.md",
"type": "filepath"
},
"123": {
"file_id": 9,
"content": "This code provides instructions to download and extract the Country211 dataset, a geolocation image classification dataset created from YFCC100m. The dataset contains balanced samples for training, validation, and testing for each country with corresponding ISO-3166 codes.",
"type": "summary"
},
"124": {
"file_id": 9,
"content": "# The Country211 Dataset\nIn the paper, we used an image classification dataset called Country211, to evaluate the model's capability on geolocation. To do so, we filtered the YFCC100m dataset that have GPS coordinate corresponding to a [ISO-3166 country code](https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes) and created a balanced dataset by sampling 150 train images, 50 validation images, and 100 test images images for each country.\nThe following command will download an 11GB archive countaining the images and extract into a subdirectory `country211`:\n```bash\nwget https://openaipublic.azureedge.net/clip/data/country211.tgz\ntar zxvf country211.tgz\n```\nThese images are a subset of the YFCC100m dataset. Use of the underlying media files is subject to the Creative Commons licenses chosen by their creators/uploaders. For more information about the YFCC100M dataset, visit [the official website](https://multimediacommons.wordpress.com/yfcc100m-core-dataset/).",
"type": "code",
"location": "/data/country211.md:1-12"
},
"125": {
"file_id": 9,
"content": "This code provides instructions to download and extract the Country211 dataset, a geolocation image classification dataset created from YFCC100m. The dataset contains balanced samples for training, validation, and testing for each country with corresponding ISO-3166 codes.",
"type": "comment"
},
"126": {
"file_id": 10,
"content": "/data/rendered-sst2.md",
"type": "filepath"
},
"127": {
"file_id": 10,
"content": "This code explains that the Rendered SST2 dataset was used in a paper for image classification. It provides instructions to download and extract the dataset using a command.",
"type": "summary"
},
"128": {
"file_id": 10,
"content": "# The Rendered SST2 Dataset\nIn the paper, we used an image classification dataset called Rendered SST2, to evaluate the model's capability on optical character recognition. To do so, we rendered the sentences in the [Standford Sentiment Treebank v2](https://nlp.stanford.edu/sentiment/treebank.html) dataset and used those as the input to the CLIP image encoder.\nThe following command will download a 131MB archive countaining the images and extract into a subdirectory `rendered-sst2`:\n```bash\nwget https://openaipublic.azureedge.net/clip/data/rendered-sst2.tgz\ntar zxvf rendered-sst2.tgz\n```",
"type": "code",
"location": "/data/rendered-sst2.md:1-10"
},
"129": {
"file_id": 10,
"content": "This code explains that the Rendered SST2 dataset was used in a paper for image classification. It provides instructions to download and extract the dataset using a command.",
"type": "comment"
},
"130": {
"file_id": 11,
"content": "/data/yfcc100m.md",
"type": "filepath"
},
"131": {
"file_id": 11,
"content": "This code is downloading and decompressing a subset of the YFCC100M dataset, which contains 14,829,396 images with English language titles and/or descriptions. The dataset's usage follows Creative Commons licenses chosen by creators/uploaders.",
"type": "summary"
},
"132": {
"file_id": 11,
"content": "# The YFCC100M Subset\nIn the paper, we performed a dataset ablation using a subset of the YFCC100M dataset and showed that the performance remained largely similar. \nThe subset contains 14,829,396 images, about 15% of the full dataset, which have been filtered to only keep those with natural languag titles and/or descriptions in English.\nWe provide the list of (line number, photo identifier, photo hash) of each image contained in this subset. These correspond to the first three columns in the dataset's metadata TSV file.\n```bash\nwget https://openaipublic.azureedge.net/clip/data/yfcc100m_subset_data.tsv.bz2\nbunzip2 yfcc100m_subset_data.tsv.bz2\n```\nUse of the underlying media files is subject to the Creative Commons licenses chosen by their creators/uploaders. For more information about the YFCC100M dataset, visit [the official website](https://multimediacommons.wordpress.com/yfcc100m-core-dataset/).",
"type": "code",
"location": "/data/yfcc100m.md:1-14"
},
"133": {
"file_id": 11,
"content": "This code is downloading and decompressing a subset of the YFCC100M dataset, which contains 14,829,396 images with English language titles and/or descriptions. The dataset's usage follows Creative Commons licenses chosen by creators/uploaders.",
"type": "comment"
},
"134": {
"file_id": 12,
"content": "/notebooks/Interacting_with_CLIP.py",
"type": "filepath"
},
"135": {
"file_id": 12,
"content": "The code imports libraries, prepares CLIP model and image datasets, calculates text-image similarity using CIFAR-100 dataset, and visualizes the relationship in a heatmap.",
"type": "summary"
},
"136": {
"file_id": 12,
"content": "#! pip install ftfy regex tqdm\n#! pip install git+https://github.com/openai/CLIP.git\nimport numpy as np\nimport torch\nfrom pkg_resources import packaging\nprint(\"Torch version:\", torch.__version__)\nimport clip\nclip.available_models()\nmodel, preprocess = clip.load(\"ViT-B/32\")\nmodel.cuda().eval()\ninput_resolution = model.visual.input_resolution\ncontext_length = model.context_length\nvocab_size = model.vocab_size\nprint(\"Model parameters:\", f\"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}\")\nprint(\"Input resolution:\", input_resolution)\nprint(\"Context length:\", context_length)\nprint(\"Vocab size:\", vocab_size)\npreprocess\nclip.tokenize(\"Hello World!\")\nimport os\nimport skimage\nimport IPython.display\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\nfrom collections import OrderedDict\nimport torch\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n# images in skimage to use and their textual descriptions\ndescriptions = {\n \"page\": \"a page of text about segmentation\",\n \"chelsea\": \"a facial photo of a tabby cat\",",
"type": "code",
"location": "/notebooks/Interacting_with_CLIP.py:1-46"
},
"137": {
"file_id": 12,
"content": "The code imports necessary libraries, checks the installed versions of PyTorch and CLIP, loads a pre-trained CLIP model with specified parameters, and defines some variables including image resolution, context length, and vocabulary size. It also displays the total number of model parameters and shows how to tokenize text using CLIP's tokenizer. The code then imports necessary libraries for image processing and visualization like skimage, IPython.display, matplotlib.pyplot, PIL, numpy, and torch. Finally, it defines a dictionary with image names as keys and their corresponding descriptions as values.",
"type": "comment"
},
"138": {
"file_id": 12,
"content": " \"astronaut\": \"a portrait of an astronaut with the American flag\",\n \"rocket\": \"a rocket standing on a launchpad\",\n \"motorcycle_right\": \"a red motorcycle standing in a garage\",\n \"camera\": \"a person looking at a camera on a tripod\",\n \"horse\": \"a black-and-white silhouette of a horse\", \n \"coffee\": \"a cup of coffee on a saucer\"\n}\noriginal_images = []\nimages = []\ntexts = []\nplt.figure(figsize=(16, 5))\nfor filename in [filename for filename in os.listdir(skimage.data_dir) if filename.endswith(\".png\") or filename.endswith(\".jpg\")]:\n name = os.path.splitext(filename)[0]\n if name not in descriptions:\n continue\n image = Image.open(os.path.join(skimage.data_dir, filename)).convert(\"RGB\")\n plt.subplot(2, 4, len(images) + 1)\n plt.imshow(image)\n plt.title(f\"{filename}\\n{descriptions[name]}\")\n plt.xticks([])\n plt.yticks([])\n original_images.append(image)\n images.append(preprocess(image))\n texts.append(descriptions[name])\nplt.tight_layout()\nimage_input = torch.tensor(np.stack(images)).cuda()",
"type": "code",
"location": "/notebooks/Interacting_with_CLIP.py:47-80"
},
"139": {
"file_id": 12,
"content": "This code is preparing a dataset of images and corresponding descriptions for CLIP. It reads image files from the specified directory, selects relevant images based on provided descriptions, preprocesses them, and stores in lists. The images are then displayed as a grid with titles showing their names and descriptions. Finally, the preprocessed images are converted to torch tensor for use with CLIP.",
"type": "comment"
},
"140": {
"file_id": 12,
"content": "text_tokens = clip.tokenize([\"This is \" + desc for desc in texts]).cuda()\nwith torch.no_grad():\n image_features = model.encode_image(image_input).float()\n text_features = model.encode_text(text_tokens).float()\nimage_features /= image_features.norm(dim=-1, keepdim=True)\ntext_features /= text_features.norm(dim=-1, keepdim=True)\nsimilarity = text_features.cpu().numpy() @ image_features.cpu().numpy().T\ncount = len(descriptions)\nplt.figure(figsize=(20, 14))\nplt.imshow(similarity, vmin=0.1, vmax=0.3)\n# plt.colorbar()\nplt.yticks(range(count), texts, fontsize=18)\nplt.xticks([])\nfor i, image in enumerate(original_images):\n plt.imshow(image, extent=(i - 0.5, i + 0.5, -1.6, -0.6), origin=\"lower\")\nfor x in range(similarity.shape[1]):\n for y in range(similarity.shape[0]):\n plt.text(x, y, f\"{similarity[y, x]:.2f}\", ha=\"center\", va=\"center\", size=12)\nfor side in [\"left\", \"top\", \"right\", \"bottom\"]:\n plt.gca().spines[side].set_visible(False)\nplt.xlim([-0.5, count - 0.5])\nplt.ylim([count + 0.5, -2])\nplt.title(\"Cosine similarity between text and image features\", size=20)",
"type": "code",
"location": "/notebooks/Interacting_with_CLIP.py:81-110"
},
"141": {
"file_id": 12,
"content": "Code chunk normalizes text and image features, calculates cosine similarity between them, and plots a heatmap to visualize the relationship.",
"type": "comment"
},
"142": {
"file_id": 12,
"content": "from torchvision.datasets import CIFAR100\ncifar100 = CIFAR100(os.path.expanduser(\"~/.cache\"), transform=preprocess, download=True)\ntext_descriptions = [f\"This is a photo of a {label}\" for label in cifar100.classes]\ntext_tokens = clip.tokenize(text_descriptions).cuda()\nwith torch.no_grad():\n text_features = model.encode_text(text_tokens).float()\n text_features /= text_features.norm(dim=-1, keepdim=True)\ntext_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)\ntop_probs, top_labels = text_probs.cpu().topk(5, dim=-1)\nplt.figure(figsize=(16, 16))\nfor i, image in enumerate(original_images):\n plt.subplot(4, 4, 2 * i + 1)\n plt.imshow(image)\n plt.axis(\"off\")\n plt.subplot(4, 4, 2 * i + 2)\n y = np.arange(top_probs.shape[-1])\n plt.grid()\n plt.barh(y, top_probs[i])\n plt.gca().invert_yaxis()\n plt.gca().set_axisbelow(True)\n plt.yticks(y, [cifar100.classes[index] for index in top_labels[i].numpy()])\n plt.xlabel(\"probability\")\nplt.subplots_adjust(wspace=0.5)\nplt.show()",
"type": "code",
"location": "/notebooks/Interacting_with_CLIP.py:112-143"
},
"143": {
"file_id": 12,
"content": "This code is loading the CIFAR-100 dataset, extracting image features and text descriptions from it, then calculating the similarity between image features and text features. The results are displayed in a visualization showing the top 5 most probable labels for each image.",
"type": "comment"
},
"144": {
"file_id": 13,
"content": "/notebooks/Prompt_Engineering_for_ImageNet.py",
"type": "filepath"
},
"145": {
"file_id": 13,
"content": "The code installs libraries, loads the CLIP model, and processes a dataset before performing zero-shot classification and calculating top-1/top-5 accuracy on ImageNet dataset.",
"type": "summary"
},
"146": {
"file_id": 13,
"content": "#! pip install ftfy regex tqdm\n#! pip install git+https://github.com/openai/CLIP.git\nimport numpy as np\nimport torch\nimport clip\nfrom tqdm.notebook import tqdm\nfrom pkg_resources import packaging\nprint(\"Torch version:\", torch.__version__)\nclip.available_models()\nmodel, preprocess = clip.load(\"ViT-B/32\")\ninput_resolution = model.visual.input_resolution\ncontext_length = model.context_length\nvocab_size = model.vocab_size\nprint(\"Model parameters:\", f\"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}\")\nprint(\"Input resolution:\", input_resolution)\nprint(\"Context length:\", context_length)\nprint(\"Vocab size:\", vocab_size)\nimport json\nimagenet_data = json.loads(open(\"imagenet_data.json\",\"r\").read())\nimagenet_classes = imagenet_data['imagenet_classes']\nimagenet_templates = imagenet_data['imagenet_templates']\nprint(f\"{len(imagenet_classes)} classes, {len(imagenet_templates)} templates\")\n# execute:\n# ! pip install git+https://github.com/modestyachts/ImageNetV2_pytorch\nfrom imagenetv2_pytorch import ImageNetV2Dataset",
"type": "code",
"location": "/notebooks/Prompt_Engineering_for_ImageNet.py:1-36"
},
"147": {
"file_id": 13,
"content": "This code installs required libraries, loads OpenAI's CLIP model, and retrieves Imagenet classes and templates. It also imports the ImageNetV2Dataset from a specific repository.",
"type": "comment"
},
"148": {
"file_id": 13,
"content": "images = ImageNetV2Dataset(transform=preprocess)\nloader = torch.utils.data.DataLoader(images, batch_size=32, num_workers=2)\ndef zeroshot_classifier(classnames, templates):\n with torch.no_grad():\n zeroshot_weights = []\n for classname in tqdm(classnames):\n texts = [template.format(classname) for template in templates] #format with class\n texts = clip.tokenize(texts).cuda() #tokenize\n class_embeddings = model.encode_text(texts) #embed with text encoder\n class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)\n class_embedding = class_embeddings.mean(dim=0)\n class_embedding /= class_embedding.norm()\n zeroshot_weights.append(class_embedding)\n zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()\n return zeroshot_weights\nzeroshot_weights = zeroshot_classifier(imagenet_classes, imagenet_templates)\ndef accuracy(output, target, topk=(1,)):\n pred = output.topk(max(topk), 1, True, True)[1].t()",
"type": "code",
"location": "/notebooks/Prompt_Engineering_for_ImageNet.py:38-59"
},
"149": {
"file_id": 13,
"content": "Code snippet performs zero-shot classification for the ImageNet dataset. It generates embeddings for given class names using text templates, averages them and stores in zeroshot_weights. The accuracy function calculates accuracy based on the output and target values.",
"type": "comment"
},
"150": {
"file_id": 13,
"content": " correct = pred.eq(target.view(1, -1).expand_as(pred))\n return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]\nwith torch.no_grad():\n top1, top5, n = 0., 0., 0.\n for i, (images, target) in enumerate(tqdm(loader)):\n images = images.cuda()\n target = target.cuda()\n # predict\n image_features = model.encode_image(images)\n image_features /= image_features.norm(dim=-1, keepdim=True)\n logits = 100. * image_features @ zeroshot_weights\n # measure accuracy\n acc1, acc5 = accuracy(logits, target, topk=(1, 5))\n top1 += acc1\n top5 += acc5\n n += images.size(0)\ntop1 = (top1 / n) * 100\ntop5 = (top5 / n) * 100 \nprint(f\"Top-1 accuracy: {top1:.2f}\")\nprint(f\"Top-5 accuracy: {top5:.2f}\")",
"type": "code",
"location": "/notebooks/Prompt_Engineering_for_ImageNet.py:60-84"
},
"151": {
"file_id": 13,
"content": "The code calculates the top-1 and top-5 accuracy of a model's predictions on ImageNet dataset. It computes the accuracy by comparing predicted probabilities with ground truth labels, averages them over all images in the batch, and prints the results.",
"type": "comment"
},
"152": {
"file_id": 14,
"content": "/tests/test_consistency.py",
"type": "filepath"
},
"153": {
"file_id": 14,
"content": "Testing consistency between JIT and non-JIT versions of CLIP model.",
"type": "summary"
},
"154": {
"file_id": 14,
"content": "import numpy as np\nimport pytest\nimport torch\nfrom PIL import Image\nimport clip\n@pytest.mark.parametrize('model_name', clip.available_models())\ndef test_consistency(model_name):\n device = \"cpu\"\n jit_model, transform = clip.load(model_name, device=device, jit=True)\n py_model, _ = clip.load(model_name, device=device, jit=False)\n image = transform(Image.open(\"CLIP.png\")).unsqueeze(0).to(device)\n text = clip.tokenize([\"a diagram\", \"a dog\", \"a cat\"]).to(device)\n with torch.no_grad():\n logits_per_image, _ = jit_model(image, text)\n jit_probs = logits_per_image.softmax(dim=-1).cpu().numpy()\n logits_per_image, _ = py_model(image, text)\n py_probs = logits_per_image.softmax(dim=-1).cpu().numpy()\n assert np.allclose(jit_probs, py_probs, atol=0.01, rtol=0.1)",
"type": "code",
"location": "/tests/test_consistency.py:1-25"
},
"155": {
"file_id": 14,
"content": "Testing consistency between JIT and non-JIT versions of CLIP model.",
"type": "comment"
}
}

10
docs/doc/0a617d1a-287b-4006-975a-7089f96cdaf9.json Normal file
View File

@ -0,0 +1,10 @@
{
"summary": "This code is downloading and decompressing a subset of the YFCC100M dataset, which contains 14,829,396 images with English language titles and/or descriptions. The dataset's usage follows Creative Commons licenses chosen by creators/uploaders.",
"details": [
{
"comment": "This code is downloading and decompressing a subset of the YFCC100M dataset, which contains 14,829,396 images with English language titles and/or descriptions. The dataset's usage follows Creative Commons licenses chosen by creators/uploaders.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/data/yfcc100m.md\":0-13",
"content": "# The YFCC100M Subset\nIn the paper, we performed a dataset ablation using a subset of the YFCC100M dataset and showed that the performance remained largely similar. \nThe subset contains 14,829,396 images, about 15% of the full dataset, which have been filtered to only keep those with natural languag titles and/or descriptions in English.\nWe provide the list of (line number, photo identifier, photo hash) of each image contained in this subset. These correspond to the first three columns in the dataset's metadata TSV file.\n```bash\nwget https://openaipublic.azureedge.net/clip/data/yfcc100m_subset_data.tsv.bz2\nbunzip2 yfcc100m_subset_data.tsv.bz2\n```\nUse of the underlying media files is subject to the Creative Commons licenses chosen by their creators/uploaders. For more information about the YFCC100M dataset, visit [the official website](https://multimediacommons.wordpress.com/yfcc100m-core-dataset/)."
}
]
}

30
docs/doc/109cc28f-60c0-4598-8d01-90151049bec8.json Normal file
View File

@ -0,0 +1,30 @@
{
"summary": "The code defines a `SimpleTokenizer` class that uses Byte Pair Encoding (BPE) for text tokenization, cleans text data, and provides encode and decode functions. It iterates through word characters to form new words by identifying bigrams and breaks when only one character remains.",
"details": [
{
"comment": "This code defines two functions: `default_bpe()` and `bytes_to_unicode()`. The `default_bpe()` function returns the path to the \"bpe_simple_vocab_16e6.txt.gz\" file, which seems to be a byte-pair encoding (BPE) vocabulary file. The `bytes_to_unicode()` function creates two lists - one containing Unicode characters from \"!\" to \"~\", and another containing characters from \"\u00a1\" to \"\u00ac\" and \"\u00ae\" to \"\u00ff\". It then iterates through all 256 possible byte values, checking if they are not in the defined character ranges. If a value is not found in these ranges, it adds it to both lists. The function aims to create lookup tables between utf-8 bytes and Unicode strings for efficient BPE encoding.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/simple_tokenizer.py\":0-29",
"content": "import gzip\nimport html\nimport os\nfrom functools import lru_cache\nimport ftfy\nimport regex as re\n@lru_cache()\ndef default_bpe():\n return os.path.join(os.path.dirname(os.path.abspath(__file__)), \"bpe_simple_vocab_16e6.txt.gz\")\n@lru_cache()\ndef bytes_to_unicode():\n \"\"\"\n Returns list of utf-8 byte and a corresponding list of unicode strings.\n The reversible bpe codes work on unicode strings.\n This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.\n When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.\n This is a signficant percentage of your normal, say, 32K bpe vocab.\n To avoid that, we want lookup tables between utf-8 bytes and unicode strings.\n And avoids mapping to whitespace/control characters the bpe code barfs on.\n \"\"\"\n bs = list(range(ord(\"!\"), ord(\"~\")+1))+list(range(ord(\"\u00a1\"), ord(\"\u00ac\")+1))+list(range(ord(\"\u00ae\"), ord(\"\u00ff\")+1))\n cs = bs[:]\n n = 0\n for b in range(2**8):\n if b not in bs:"
},
{
"comment": "This code defines a class `SimpleTokenizer` that performs text tokenization using Byte Pair Encoding (BPE). It also includes functions for cleaning text data, such as removing special characters, fixing text, and handling whitespace.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/simple_tokenizer.py\":30-67",
"content": " bs.append(b)\n cs.append(2**8+n)\n n += 1\n cs = [chr(n) for n in cs]\n return dict(zip(bs, cs))\ndef get_pairs(word):\n \"\"\"Return set of symbol pairs in a word.\n Word is represented as tuple of symbols (symbols being variable-length strings).\n \"\"\"\n pairs = set()\n prev_char = word[0]\n for char in word[1:]:\n pairs.add((prev_char, char))\n prev_char = char\n return pairs\ndef basic_clean(text):\n text = ftfy.fix_text(text)\n text = html.unescape(html.unescape(text))\n return text.strip()\ndef whitespace_clean(text):\n text = re.sub(r'\\s+', ' ', text)\n text = text.strip()\n return text\nclass SimpleTokenizer(object):\n def __init__(self, bpe_path: str = default_bpe()):\n self.byte_encoder = bytes_to_unicode()\n self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}\n merges = gzip.open(bpe_path).read().decode(\"utf-8\").split('\\n')\n merges = merges[1:49152-256-2+1]\n merges = [tuple(merge.split()) for merge in merges]"
},
{
"comment": "This code defines a class for tokenizing text using Byte Pair Encoding (BPE). It creates the vocabulary, encoder and decoder dictionaries, initializes BPE ranks and cache. The bpe method takes a token, checks if it's in the cache, and if not, applies BPE until it reaches a single character or an existing BPE word.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/simple_tokenizer.py\":68-90",
"content": " vocab = list(bytes_to_unicode().values())\n vocab = vocab + [v+'</w>' for v in vocab]\n for merge in merges:\n vocab.append(''.join(merge))\n vocab.extend(['<|startoftext|>', '<|endoftext|>'])\n self.encoder = dict(zip(vocab, range(len(vocab))))\n self.decoder = {v: k for k, v in self.encoder.items()}\n self.bpe_ranks = dict(zip(merges, range(len(merges))))\n self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}\n self.pat = re.compile(r\"\"\"<\\|startoftext\\|>|<\\|endoftext\\|>|'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+\"\"\", re.IGNORECASE)\n def bpe(self, token):\n if token in self.cache:\n return self.cache[token]\n word = tuple(token[:-1]) + ( token[-1] + '</w>',)\n pairs = get_pairs(word)\n if not pairs:\n return token+'</w>'\n while True:\n bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))\n if bigram not in self.bpe_ranks:"
},
{
"comment": "Iterates through word characters and forms a new word by identifying bigrams (pairs of consecutive characters), joining single characters, and breaking when only one character remains. Stores the result in self.cache after converting it to a string.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/simple_tokenizer.py\":91-123",
"content": " break\n first, second = bigram\n new_word = []\n i = 0\n while i < len(word):\n try:\n j = word.index(first, i)\n new_word.extend(word[i:j])\n i = j\n except:\n new_word.extend(word[i:])\n break\n if word[i] == first and i < len(word)-1 and word[i+1] == second:\n new_word.append(first+second)\n i += 2\n else:\n new_word.append(word[i])\n i += 1\n new_word = tuple(new_word)\n word = new_word\n if len(word) == 1:\n break\n else:\n pairs = get_pairs(word)\n word = ' '.join(word)\n self.cache[token] = word\n return word\n def encode(self, text):\n bpe_tokens = []\n text = whitespace_clean(basic_clean(text)).lower()\n for token in re.findall(self.pat, text):"
},
{
"comment": "This code defines a class for tokenization using byte encoding, BPE (Byte Pair Encoding), and provides decode function. It encodes a token into its byte representation, splits it with BPE, and stores the resulting tokens in bpe_tokens list. The decode method reconstructs the original text from the token list using the decoder mapping.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/simple_tokenizer.py\":124-131",
"content": " token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))\n bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))\n return bpe_tokens\n def decode(self, tokens):\n text = ''.join([self.decoder[token] for token in tokens])\n text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=\"replace\").replace('</w>', ' ')\n return text"
}
]
}

10
docs/doc/3d0530ce-088e-4032-8983-b81ad9904b59.json Normal file
View File

@ -0,0 +1,10 @@
{
"summary": "This code explains that the Rendered SST2 dataset was used in a paper for image classification. It provides instructions to download and extract the dataset using a command.",
"details": [
{
"comment": "This code explains that the Rendered SST2 dataset was used in a paper for image classification. It provides instructions to download and extract the dataset using a command.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/data/rendered-sst2.md\":0-9",
"content": "# The Rendered SST2 Dataset\nIn the paper, we used an image classification dataset called Rendered SST2, to evaluate the model's capability on optical character recognition. To do so, we rendered the sentences in the [Standford Sentiment Treebank v2](https://nlp.stanford.edu/sentiment/treebank.html) dataset and used those as the input to the CLIP image encoder.\nThe following command will download a 131MB archive countaining the images and extract into a subdirectory `rendered-sst2`:\n```bash\nwget https://openaipublic.azureedge.net/clip/data/rendered-sst2.tgz\ntar zxvf rendered-sst2.tgz\n```"
}
]
}

55
docs/doc/48bc3c2d-e674-464e-99e9-55de2e8c3b57.json Normal file
View File

@ -0,0 +1,55 @@
{
"summary": "The code downloads and handles pre-trained models, including CLIP, checks availability, loads optimized versions, and implements device-specific patches while tokenizing input strings. This function encodes a list of texts using a tokenizer, adds start/end tokens, returns tensor, and optionally truncates if exceeding context length.",
"details": [
{
"comment": "Importing necessary libraries and defining variables for model choices and tokenizer.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":0-32",
"content": "import hashlib\nimport os\nimport urllib\nimport warnings\nfrom typing import Any, Union, List\nfrom pkg_resources import packaging\nimport torch\nfrom PIL import Image\nfrom torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize\nfrom tqdm import tqdm\nfrom .model import build_model\nfrom .simple_tokenizer import SimpleTokenizer as _Tokenizer\ntry:\n from torchvision.transforms import InterpolationMode\n BICUBIC = InterpolationMode.BICUBIC\nexcept ImportError:\n BICUBIC = Image.BICUBIC\nif packaging.version.parse(torch.__version__) < packaging.version.parse(\"1.7.1\"):\n warnings.warn(\"PyTorch version 1.7.1 or higher is recommended\")\n__all__ = [\"available_models\", \"load\", \"tokenize\"]\n_tokenizer = _Tokenizer()\n_MODELS = {\n \"RN50\": \"https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt\",\n \"RN101\": \"https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt\",\n "
},
{
"comment": "This code defines a dictionary of URLs for different pre-trained models and includes a function _download() to download the model files.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":32-42",
"content": "\"RN50x4\": \"https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt\",\n \"RN50x16\": \"https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt\",\n \"RN50x64\": \"https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt\",\n \"ViT-B/32\": \"https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt\",\n \"ViT-B/16\": \"https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt\",\n \"ViT-L/14\": \"https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt\",\n \"ViT-L/14@336px\": \"https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt\",\n}\ndef _download(url: str, root: str):"
},
{
"comment": "Creates a directory and checks if the file already exists, then downloads or verifies the file's SHA256 checksum.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":43-66",
"content": " os.makedirs(root, exist_ok=True)\n filename = os.path.basename(url)\n expected_sha256 = url.split(\"/\")[-2]\n download_target = os.path.join(root, filename)\n if os.path.exists(download_target) and not os.path.isfile(download_target):\n raise RuntimeError(f\"{download_target} exists and is not a regular file\")\n if os.path.isfile(download_target):\n if hashlib.sha256(open(download_target, \"rb\").read()).hexdigest() == expected_sha256:\n return download_target\n else:\n warnings.warn(f\"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file\")\n with urllib.request.urlopen(url) as source, open(download_target, \"wb\") as output:\n with tqdm(total=int(source.info().get(\"Content-Length\")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop:\n while True:\n buffer = source.read(8192)\n if not buffer:\n break\n output.write(buffer)\n loop.update(len(buffer))"
},
{
"comment": "Code checks the SHA256 checksum of a downloaded model and raises an error if it doesn't match. It also defines functions for image transformation, loading available CLIP models, and converting images to RGB format.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":68-101",
"content": " if hashlib.sha256(open(download_target, \"rb\").read()).hexdigest() != expected_sha256:\n raise RuntimeError(\"Model has been downloaded but the SHA256 checksum does not not match\")\n return download_target\ndef _convert_image_to_rgb(image):\n return image.convert(\"RGB\")\ndef _transform(n_px):\n return Compose([\n Resize(n_px, interpolation=BICUBIC),\n CenterCrop(n_px),\n _convert_image_to_rgb,\n ToTensor(),\n Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),\n ])\ndef available_models() -> List[str]:\n \"\"\"Returns the names of available CLIP models\"\"\"\n return list(_MODELS.keys())\ndef load(name: str, device: Union[str, torch.device] = \"cuda\" if torch.cuda.is_available() else \"cpu\", jit: bool = False, download_root: str = None):\n \"\"\"Load a CLIP model\n Parameters\n ----------\n name : str\n A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict\n device : Union[str, torch.device]"
},
{
"comment": "The code downloads the CLIP model based on the specified name and device. It checks if the model is available as a file or downloads it from the provided root path. The JIT-optimized version of the model is loaded if 'jit' is True, otherwise, the non-JIT version is used.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":102-130",
"content": " The device to put the loaded model\n jit : bool\n Whether to load the optimized JIT model or more hackable non-JIT model (default).\n download_root: str\n path to download the model files; by default, it uses \"~/.cache/clip\"\n Returns\n -------\n model : torch.nn.Module\n The CLIP model\n preprocess : Callable[[PIL.Image], torch.Tensor]\n A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input\n \"\"\"\n if name in _MODELS:\n model_path = _download(_MODELS[name], download_root or os.path.expanduser(\"~/.cache/clip\"))\n elif os.path.isfile(name):\n model_path = name\n else:\n raise RuntimeError(f\"Model {name} not found; available models = {available_models()}\")\n with open(model_path, 'rb') as opened_file:\n try:\n # loading JIT archive\n model = torch.jit.load(opened_file, map_location=device if jit else \"cpu\").eval()\n state_dict = None\n except RuntimeError:"
},
{
"comment": "Loading saved state dict and handling JIT (Just-In-Time) support.\nIf not a JIT archive, loading as a state dict instead.\nLoading model with or without JIT support depending on jit variable.\nConverting model to float if device is CPU.\nReturning the model and transformed input resolution.\nPatching device names using torch.jit.trace.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":131-156",
"content": " # loading saved state dict\n if jit:\n warnings.warn(f\"File {model_path} is not a JIT archive. Loading as a state dict instead\")\n jit = False\n state_dict = torch.load(opened_file, map_location=\"cpu\")\n if not jit:\n model = build_model(state_dict or model.state_dict()).to(device)\n if str(device) == \"cpu\":\n model.float()\n return model, _transform(model.visual.input_resolution)\n # patch the device names\n device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])\n device_node = [n for n in device_holder.graph.findAllNodes(\"prim::Constant\") if \"Device\" in repr(n)][-1]\n def _node_get(node: torch._C.Node, key: str):\n \"\"\"Gets attributes of a node which is polymorphic over return type.\n From https://github.com/pytorch/pytorch/pull/82628\n \"\"\"\n sel = node.kindOf(key)\n return getattr(node, sel)(key)\n def patch_device(module):\n try:"
},
{
"comment": "Applying device-specific patches to the model's graph.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":157-183",
"content": " graphs = [module.graph] if hasattr(module, \"graph\") else []\n except RuntimeError:\n graphs = []\n if hasattr(module, \"forward1\"):\n graphs.append(module.forward1.graph)\n for graph in graphs:\n for node in graph.findAllNodes(\"prim::Constant\"):\n if \"value\" in node.attributeNames() and str(_node_get(node, \"value\")).startswith(\"cuda\"):\n node.copyAttributes(device_node)\n model.apply(patch_device)\n patch_device(model.encode_image)\n patch_device(model.encode_text)\n # patch dtype to float32 on CPU\n if str(device) == \"cpu\":\n float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])\n float_input = list(float_holder.graph.findNode(\"aten::to\").inputs())[1]\n float_node = float_input.node()\n def patch_float(module):\n try:\n graphs = [module.graph] if hasattr(module, \"graph\") else []\n except RuntimeError:\n graphs = []"
},
{
"comment": "The code is applying a patch to select functions in the model to convert certain types to floats, then tokenizes input strings based on context_length.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":185-213",
"content": " if hasattr(module, \"forward1\"):\n graphs.append(module.forward1.graph)\n for graph in graphs:\n for node in graph.findAllNodes(\"aten::to\"):\n inputs = list(node.inputs())\n for i in [1, 2]: # dtype can be the second or third argument to aten::to()\n if _node_get(inputs[i].node(), \"value\") == 5:\n inputs[i].node().copyAttributes(float_node)\n model.apply(patch_float)\n patch_float(model.encode_image)\n patch_float(model.encode_text)\n model.float()\n return model, _transform(model.input_resolution.item())\ndef tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]:\n \"\"\"\n Returns the tokenized representation of given input string(s)\n Parameters\n ----------\n texts : Union[str, List[str]]\n An input string or a list of input strings to tokenize\n context_length : int"
},
{
"comment": "This function takes a list of texts and encodes them using the tokenizer. It then adds start and end of text tokens, and returns a tensor of shape [number of input strings, context_length]. If torch version is <1.8.0, it uses LongTensor for indices.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":214-236",
"content": " The context length to use; all CLIP models use 77 as the context length\n truncate: bool\n Whether to truncate the text in case its encoding is longer than the context length\n Returns\n -------\n A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length].\n We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long.\n \"\"\"\n if isinstance(texts, str):\n texts = [texts]\n sot_token = _tokenizer.encoder[\"<|startoftext|>\"]\n eot_token = _tokenizer.encoder[\"<|endoftext|>\"]\n all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]\n if packaging.version.parse(torch.__version__) < packaging.version.parse(\"1.8.0\"):\n result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)\n else:\n result = torch.zeros(len(all_tokens), context_length, dtype=torch.int)\n for i, tokens in enumerate(all_tokens):\n if len(tokens) > context_length:"
},
{
"comment": "If truncate is True, only keep the first context_length tokens and set the last token to eot_token. If not, raise an error if input text exceeds the context length. Store the tokens as a torch tensor in result.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/clip.py\":237-244",
"content": " if truncate:\n tokens = tokens[:context_length]\n tokens[-1] = eot_token\n else:\n raise RuntimeError(f\"Input {texts[i]} is too long for context length {context_length}\")\n result[i, :len(tokens)] = torch.tensor(tokens)\n return result"
}
]
}

10
docs/doc/5492ae61-69c2-4380-b6d5-4d34f19439b1.json Normal file
View File

@ -0,0 +1,10 @@
{
"summary": "Installing required packages: ftfy, regex, tqdm, torch, and torchvision.",
"details": [
{
"comment": "Installing required packages: ftfy, regex, tqdm, torch, and torchvision.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/requirements.txt\":0-4",
"content": "ftfy\nregex\ntqdm\ntorch\ntorchvision"
}
]
}

10
docs/doc/5a60b96f-dc4f-4080-807b-267a59e9850a.json Normal file
View File

@ -0,0 +1,10 @@
{
"summary": "This code sets up a Python package named \"clip\" using setuptools. It imports necessary modules, defines package attributes and requirements, and specifies installation dependencies. It also includes the \"dev\" extra requirement for developers.",
"details": [
{
"comment": "This code sets up a Python package named \"clip\" using setuptools. It imports necessary modules, defines package attributes and requirements, and specifies installation dependencies. It also includes the \"dev\" extra requirement for developers.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/setup.py\":0-20",
"content": "import os\nimport pkg_resources\nfrom setuptools import setup, find_packages\nsetup(\n name=\"clip\",\n py_modules=[\"clip\"],\n version=\"1.0\",\n description=\"\",\n author=\"OpenAI\",\n packages=find_packages(exclude=[\"tests*\"]),\n install_requires=[\n str(r)\n for r in pkg_resources.parse_requirements(\n open(os.path.join(os.path.dirname(__file__), \"requirements.txt\"))\n )\n ],\n include_package_data=True,\n extras_require={'dev': ['pytest']},\n)"
}
]
}

40
docs/doc/5beea2a7-a8f8-4af2-b551-48ea61ffdce6.json Normal file
View File

@ -0,0 +1,40 @@
{
"summary": "Both comments discuss using CLIP, a pre-trained model, for image and text feature extraction. Comment A focuses on CIFAR100 feature extraction and similarity computation, while Comment B covers logistic regression implementation, hyperparameter tuning, and utilizes OpenCLIP and Hugging Face CLIP implementations.",
"details": [
{
"comment": "The code provides a brief introduction to CLIP, a neural network trained on various image-text pairs. It can predict relevant text based on an image without directly optimizing for the task and matches the performance of ResNet50 on ImageNet \"zero-shot\" without using any labeled examples. The code also explains how to install necessary dependencies and set up the environment to use CLIP.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/README.md\":0-16",
"content": "# CLIP\n[[Blog]](https://openai.com/blog/clip/) [[Paper]](https://arxiv.org/abs/2103.00020) [[Model Card]](model-card.md) [[Colab]](https://colab.research.google.com/github/openai/clip/blob/master/notebooks/Interacting_with_CLIP.ipynb)\nCLIP (Contrastive Language-Image Pre-Training) is a neural network trained on a variety of (image, text) pairs. It can be instructed in natural language to predict the most relevant text snippet, given an image, without directly optimizing for the task, similarly to the zero-shot capabilities of GPT-2 and 3. We found CLIP matches the performance of the original ResNet50 on ImageNet \u201czero-shot\u201d without using any of the original 1.28M labeled examples, overcoming several major challenges in computer vision.\n## Approach\n![CLIP](CLIP.png)\n## Usage\nFirst, [install PyTorch 1.7.1](https://pytorch.org/get-started/locally/) (or later) and torchvision, as well as small additional dependencies, and then install this repo as a Python package. On a CUDA GPU machine, the following will do the trick:"
},
{
"comment": "Code installs necessary packages for running CLIP and loads the model.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/README.md\":18-54",
"content": "```bash\n$ conda install --yes -c pytorch pytorch=1.7.1 torchvision cudatoolkit=11.0\n$ pip install ftfy regex tqdm\n$ pip install git+https://github.com/openai/CLIP.git\n```\nReplace `cudatoolkit=11.0` above with the appropriate CUDA version on your machine or `cpuonly` when installing on a machine without a GPU.\n```python\nimport torch\nimport clip\nfrom PIL import Image\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel, preprocess = clip.load(\"ViT-B/32\", device=device)\nimage = preprocess(Image.open(\"CLIP.png\")).unsqueeze(0).to(device)\ntext = clip.tokenize([\"a diagram\", \"a dog\", \"a cat\"]).to(device)\nwith torch.no_grad():\n image_features = model.encode_image(image)\n text_features = model.encode_text(text)\n logits_per_image, logits_per_text = model(image, text)\n probs = logits_per_image.softmax(dim=-1).cpu().numpy()\nprint(\"Label probs:\", probs) # prints: [[0.9927937 0.00421068 0.00299572]]\n```\n## API\nThe CLIP module `clip` provides the following methods:\n#### `clip.available_models()`\nReturns the names of the available CLIP models."
},
{
"comment": "This code snippet is for the CLIP library, which provides a model for visual-textual similarity. It includes two primary functions: `clip.load()` and `clip.tokenize()`. The `clip.load()` function loads a pre-trained CLIP model specified by the `name` parameter or downloads it if necessary. The `clip.tokenize()` function tokenizes input text(s) and returns LongTensor containing the tokenized sequences. The loaded model also supports two methods: `model.encode_image()` to encode image features and `model.encode_text()` to encode text features.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/README.md\":56-76",
"content": "#### `clip.load(name, device=..., jit=False)`\nReturns the model and the TorchVision transform needed by the model, specified by the model name returned by `clip.available_models()`. It will download the model as necessary. The `name` argument can also be a path to a local checkpoint.\nThe device to run the model can be optionally specified, and the default is to use the first CUDA device if there is any, otherwise the CPU. When `jit` is `False`, a non-JIT version of the model will be loaded.\n#### `clip.tokenize(text: Union[str, List[str]], context_length=77)`\nReturns a LongTensor containing tokenized sequences of given text input(s). This can be used as the input to the model\n---\nThe model returned by `clip.load()` supports the following methods:\n#### `model.encode_image(image: Tensor)`\nGiven a batch of images, returns the image features encoded by the vision portion of the CLIP model.\n#### `model.encode_text(text: Tensor)`\nGiven a batch of text tokens, returns the text features encoded by the language portion of the CLIP model."
},
{
"comment": "The code snippet loads the CLIP model (ViT-B/32) and prepares inputs for zero-shot prediction using an image from the CIFAR-100 dataset.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/README.md\":78-105",
"content": "#### `model(image: Tensor, text: Tensor)`\nGiven a batch of images and a batch of text tokens, returns two Tensors, containing the logit scores corresponding to each image and text input. The values are cosine similarities between the corresponding image and text features, times 100.\n## More Examples\n### Zero-Shot Prediction\nThe code below performs zero-shot prediction using CLIP, as shown in Appendix B in the paper. This example takes an image from the [CIFAR-100 dataset](https://www.cs.toronto.edu/~kriz/cifar.html), and predicts the most likely labels among the 100 textual labels from the dataset.\n```python\nimport os\nimport clip\nimport torch\nfrom torchvision.datasets import CIFAR100\n# Load the model\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel, preprocess = clip.load('ViT-B/32', device)\n# Download the dataset\ncifar100 = CIFAR100(root=os.path.expanduser(\"~/.cache\"), download=True, train=False)\n# Prepare the inputs\nimage, class_id = cifar100[3637]\nimage_input = preprocess(image).unsqueeze(0).to(device)"
},
{
"comment": "This code calculates the similarity between image features and text features using dot product and softmax, then prints the top 5 most similar labels for the given image.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/README.md\":106-137",
"content": "text_inputs = torch.cat([clip.tokenize(f\"a photo of a {c}\") for c in cifar100.classes]).to(device)\n# Calculate features\nwith torch.no_grad():\n image_features = model.encode_image(image_input)\n text_features = model.encode_text(text_inputs)\n# Pick the top 5 most similar labels for the image\nimage_features /= image_features.norm(dim=-1, keepdim=True)\ntext_features /= text_features.norm(dim=-1, keepdim=True)\nsimilarity = (100.0 * image_features @ text_features.T).softmax(dim=-1)\nvalues, indices = similarity[0].topk(5)\n# Print the result\nprint(\"\\nTop predictions:\\n\")\nfor value, index in zip(values, indices):\n print(f\"{cifar100.classes[index]:>16s}: {100 * value.item():.2f}%\")\n```\nThe output will look like the following (the exact numbers may be slightly different depending on the compute device):\n```\nTop predictions:\n snake: 65.31%\n turtle: 12.29%\n sweet_pepper: 3.83%\n lizard: 1.88%\n crocodile: 1.75%\n```\nNote that this example uses the `encode_image()` and `encode_text()` methods that return the encoded features of given inputs."
},
{
"comment": "This code loads a pre-trained CLIP model, then applies it to CIFAR100 dataset for feature extraction using logistic regression.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/README.md\":140-176",
"content": "### Linear-probe evaluation\nThe example below uses [scikit-learn](https://scikit-learn.org/) to perform logistic regression on image features.\n```python\nimport os\nimport clip\nimport torch\nimport numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom torch.utils.data import DataLoader\nfrom torchvision.datasets import CIFAR100\nfrom tqdm import tqdm\n# Load the model\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel, preprocess = clip.load('ViT-B/32', device)\n# Load the dataset\nroot = os.path.expanduser(\"~/.cache\")\ntrain = CIFAR100(root, download=True, train=True, transform=preprocess)\ntest = CIFAR100(root, download=True, train=False, transform=preprocess)\ndef get_features(dataset):\n all_features = []\n all_labels = []\n with torch.no_grad():\n for images, labels in tqdm(DataLoader(dataset, batch_size=100)):\n features = model.encode_image(images.to(device))\n all_features.append(features)\n all_labels.append(labels)\n return torch.cat(all_features).cpu().numpy(), torch.cat(all_labels).cpu().numpy()"
},
{
"comment": "Calculating image features, performing logistic regression, and evaluating using a logistic regression classifier.\nHyperparameter C should be determined via validation split.\nSee also OpenCLIP and Hugging Face CLIP implementation.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/README.md\":178-198",
"content": "# Calculate the image features\ntrain_features, train_labels = get_features(train)\ntest_features, test_labels = get_features(test)\n# Perform logistic regression\nclassifier = LogisticRegression(random_state=0, C=0.316, max_iter=1000, verbose=1)\nclassifier.fit(train_features, train_labels)\n# Evaluate using the logistic regression classifier\npredictions = classifier.predict(test_features)\naccuracy = np.mean((test_labels == predictions).astype(float)) * 100.\nprint(f\"Accuracy = {accuracy:.3f}\")\n```\nNote that the `C` value should be determined via a hyperparameter sweep using a validation split.\n## See Also\n* [OpenCLIP](https://github.com/mlfoundations/open_clip): includes larger and independently trained CLIP models up to ViT-G/14\n* [Hugging Face implementation of CLIP](https://huggingface.co/docs/transformers/model_doc/clip): for easier integration with the HF ecosystem"
}
]
}

10
docs/doc/7099cdac-3f70-419a-9bb2-5165ad46103f.json Normal file
View File

@ -0,0 +1,10 @@
{
"summary": "Testing consistency between JIT and non-JIT versions of CLIP model.",
"details": [
{
"comment": "Testing consistency between JIT and non-JIT versions of CLIP model.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/tests/test_consistency.py\":0-24",
"content": "import numpy as np\nimport pytest\nimport torch\nfrom PIL import Image\nimport clip\n@pytest.mark.parametrize('model_name', clip.available_models())\ndef test_consistency(model_name):\n device = \"cpu\"\n jit_model, transform = clip.load(model_name, device=device, jit=True)\n py_model, _ = clip.load(model_name, device=device, jit=False)\n image = transform(Image.open(\"CLIP.png\")).unsqueeze(0).to(device)\n text = clip.tokenize([\"a diagram\", \"a dog\", \"a cat\"]).to(device)\n with torch.no_grad():\n logits_per_image, _ = jit_model(image, text)\n jit_probs = logits_per_image.softmax(dim=-1).cpu().numpy()\n logits_per_image, _ = py_model(image, text)\n py_probs = logits_per_image.softmax(dim=-1).cpu().numpy()\n assert np.allclose(jit_probs, py_probs, atol=0.01, rtol=0.1)"
}
]
}

10
docs/doc/8c1fd3e7-46bb-48ba-b47b-64da66e55295.json Normal file
View File

@ -0,0 +1,10 @@
{
"summary": "Imports all functions and classes from the \"clip\" module.",
"details": [
{
"comment": "Imports all functions and classes from the \"clip\" module.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/__init__.py\":0-0",
"content": "from .clip import *"
}
]
}

20
docs/doc/a41de9d4-c920-4a82-a8f4-30bc23729dbd.json Normal file
View File

@ -0,0 +1,20 @@
{
"summary": "The code installs libraries, loads the CLIP model, and processes a dataset before performing zero-shot classification and calculating top-1/top-5 accuracy on ImageNet dataset.",
"details": [
{
"comment": "This code installs required libraries, loads OpenAI's CLIP model, and retrieves Imagenet classes and templates. It also imports the ImageNetV2Dataset from a specific repository.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/notebooks/Prompt_Engineering_for_ImageNet.py\":0-35",
"content": "#! pip install ftfy regex tqdm\n#! pip install git+https://github.com/openai/CLIP.git\nimport numpy as np\nimport torch\nimport clip\nfrom tqdm.notebook import tqdm\nfrom pkg_resources import packaging\nprint(\"Torch version:\", torch.__version__)\nclip.available_models()\nmodel, preprocess = clip.load(\"ViT-B/32\")\ninput_resolution = model.visual.input_resolution\ncontext_length = model.context_length\nvocab_size = model.vocab_size\nprint(\"Model parameters:\", f\"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}\")\nprint(\"Input resolution:\", input_resolution)\nprint(\"Context length:\", context_length)\nprint(\"Vocab size:\", vocab_size)\nimport json\nimagenet_data = json.loads(open(\"imagenet_data.json\",\"r\").read())\nimagenet_classes = imagenet_data['imagenet_classes']\nimagenet_templates = imagenet_data['imagenet_templates']\nprint(f\"{len(imagenet_classes)} classes, {len(imagenet_templates)} templates\")\n# execute:\n# ! pip install git+https://github.com/modestyachts/ImageNetV2_pytorch\nfrom imagenetv2_pytorch import ImageNetV2Dataset"
},
{
"comment": "Code snippet performs zero-shot classification for the ImageNet dataset. It generates embeddings for given class names using text templates, averages them and stores in zeroshot_weights. The accuracy function calculates accuracy based on the output and target values.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/notebooks/Prompt_Engineering_for_ImageNet.py\":37-58",
"content": "images = ImageNetV2Dataset(transform=preprocess)\nloader = torch.utils.data.DataLoader(images, batch_size=32, num_workers=2)\ndef zeroshot_classifier(classnames, templates):\n with torch.no_grad():\n zeroshot_weights = []\n for classname in tqdm(classnames):\n texts = [template.format(classname) for template in templates] #format with class\n texts = clip.tokenize(texts).cuda() #tokenize\n class_embeddings = model.encode_text(texts) #embed with text encoder\n class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)\n class_embedding = class_embeddings.mean(dim=0)\n class_embedding /= class_embedding.norm()\n zeroshot_weights.append(class_embedding)\n zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()\n return zeroshot_weights\nzeroshot_weights = zeroshot_classifier(imagenet_classes, imagenet_templates)\ndef accuracy(output, target, topk=(1,)):\n pred = output.topk(max(topk), 1, True, True)[1].t()"
},
{
"comment": "The code calculates the top-1 and top-5 accuracy of a model's predictions on ImageNet dataset. It computes the accuracy by comparing predicted probabilities with ground truth labels, averages them over all images in the batch, and prints the results.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/notebooks/Prompt_Engineering_for_ImageNet.py\":59-83",
"content": " correct = pred.eq(target.view(1, -1).expand_as(pred))\n return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]\nwith torch.no_grad():\n top1, top5, n = 0., 0., 0.\n for i, (images, target) in enumerate(tqdm(loader)):\n images = images.cuda()\n target = target.cuda()\n # predict\n image_features = model.encode_image(images)\n image_features /= image_features.norm(dim=-1, keepdim=True)\n logits = 100. * image_features @ zeroshot_weights\n # measure accuracy\n acc1, acc5 = accuracy(logits, target, topk=(1, 5))\n top1 += acc1\n top5 += acc5\n n += images.size(0)\ntop1 = (top1 / n) * 100\ntop5 = (top5 / n) * 100 \nprint(f\"Top-1 accuracy: {top1:.2f}\")\nprint(f\"Top-5 accuracy: {top5:.2f}\")"
}
]
}

25
docs/doc/a90273b5-2a4d-4bb8-9de5-aa6b7026cab6.json Normal file
View File

@ -0,0 +1,25 @@
{
"summary": "The code imports libraries, prepares CLIP model and image datasets, calculates text-image similarity using CIFAR-100 dataset, and visualizes the relationship in a heatmap.",
"details": [
{
"comment": "The code imports necessary libraries, checks the installed versions of PyTorch and CLIP, loads a pre-trained CLIP model with specified parameters, and defines some variables including image resolution, context length, and vocabulary size. It also displays the total number of model parameters and shows how to tokenize text using CLIP's tokenizer. The code then imports necessary libraries for image processing and visualization like skimage, IPython.display, matplotlib.pyplot, PIL, numpy, and torch. Finally, it defines a dictionary with image names as keys and their corresponding descriptions as values.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/notebooks/Interacting_with_CLIP.py\":0-45",
"content": "#! pip install ftfy regex tqdm\n#! pip install git+https://github.com/openai/CLIP.git\nimport numpy as np\nimport torch\nfrom pkg_resources import packaging\nprint(\"Torch version:\", torch.__version__)\nimport clip\nclip.available_models()\nmodel, preprocess = clip.load(\"ViT-B/32\")\nmodel.cuda().eval()\ninput_resolution = model.visual.input_resolution\ncontext_length = model.context_length\nvocab_size = model.vocab_size\nprint(\"Model parameters:\", f\"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}\")\nprint(\"Input resolution:\", input_resolution)\nprint(\"Context length:\", context_length)\nprint(\"Vocab size:\", vocab_size)\npreprocess\nclip.tokenize(\"Hello World!\")\nimport os\nimport skimage\nimport IPython.display\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\nfrom collections import OrderedDict\nimport torch\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n# images in skimage to use and their textual descriptions\ndescriptions = {\n \"page\": \"a page of text about segmentation\",\n \"chelsea\": \"a facial photo of a tabby cat\","
},
{
"comment": "This code is preparing a dataset of images and corresponding descriptions for CLIP. It reads image files from the specified directory, selects relevant images based on provided descriptions, preprocesses them, and stores in lists. The images are then displayed as a grid with titles showing their names and descriptions. Finally, the preprocessed images are converted to torch tensor for use with CLIP.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/notebooks/Interacting_with_CLIP.py\":46-79",
"content": " \"astronaut\": \"a portrait of an astronaut with the American flag\",\n \"rocket\": \"a rocket standing on a launchpad\",\n \"motorcycle_right\": \"a red motorcycle standing in a garage\",\n \"camera\": \"a person looking at a camera on a tripod\",\n \"horse\": \"a black-and-white silhouette of a horse\", \n \"coffee\": \"a cup of coffee on a saucer\"\n}\noriginal_images = []\nimages = []\ntexts = []\nplt.figure(figsize=(16, 5))\nfor filename in [filename for filename in os.listdir(skimage.data_dir) if filename.endswith(\".png\") or filename.endswith(\".jpg\")]:\n name = os.path.splitext(filename)[0]\n if name not in descriptions:\n continue\n image = Image.open(os.path.join(skimage.data_dir, filename)).convert(\"RGB\")\n plt.subplot(2, 4, len(images) + 1)\n plt.imshow(image)\n plt.title(f\"{filename}\\n{descriptions[name]}\")\n plt.xticks([])\n plt.yticks([])\n original_images.append(image)\n images.append(preprocess(image))\n texts.append(descriptions[name])\nplt.tight_layout()\nimage_input = torch.tensor(np.stack(images)).cuda()"
},
{
"comment": "Code chunk normalizes text and image features, calculates cosine similarity between them, and plots a heatmap to visualize the relationship.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/notebooks/Interacting_with_CLIP.py\":80-109",
"content": "text_tokens = clip.tokenize([\"This is \" + desc for desc in texts]).cuda()\nwith torch.no_grad():\n image_features = model.encode_image(image_input).float()\n text_features = model.encode_text(text_tokens).float()\nimage_features /= image_features.norm(dim=-1, keepdim=True)\ntext_features /= text_features.norm(dim=-1, keepdim=True)\nsimilarity = text_features.cpu().numpy() @ image_features.cpu().numpy().T\ncount = len(descriptions)\nplt.figure(figsize=(20, 14))\nplt.imshow(similarity, vmin=0.1, vmax=0.3)\n# plt.colorbar()\nplt.yticks(range(count), texts, fontsize=18)\nplt.xticks([])\nfor i, image in enumerate(original_images):\n plt.imshow(image, extent=(i - 0.5, i + 0.5, -1.6, -0.6), origin=\"lower\")\nfor x in range(similarity.shape[1]):\n for y in range(similarity.shape[0]):\n plt.text(x, y, f\"{similarity[y, x]:.2f}\", ha=\"center\", va=\"center\", size=12)\nfor side in [\"left\", \"top\", \"right\", \"bottom\"]:\n plt.gca().spines[side].set_visible(False)\nplt.xlim([-0.5, count - 0.5])\nplt.ylim([count + 0.5, -2])\nplt.title(\"Cosine similarity between text and image features\", size=20)"
},
{
"comment": "This code is loading the CIFAR-100 dataset, extracting image features and text descriptions from it, then calculating the similarity between image features and text features. The results are displayed in a visualization showing the top 5 most probable labels for each image.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/notebooks/Interacting_with_CLIP.py\":111-142",
"content": "from torchvision.datasets import CIFAR100\ncifar100 = CIFAR100(os.path.expanduser(\"~/.cache\"), transform=preprocess, download=True)\ntext_descriptions = [f\"This is a photo of a {label}\" for label in cifar100.classes]\ntext_tokens = clip.tokenize(text_descriptions).cuda()\nwith torch.no_grad():\n text_features = model.encode_text(text_tokens).float()\n text_features /= text_features.norm(dim=-1, keepdim=True)\ntext_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)\ntop_probs, top_labels = text_probs.cpu().topk(5, dim=-1)\nplt.figure(figsize=(16, 16))\nfor i, image in enumerate(original_images):\n plt.subplot(4, 4, 2 * i + 1)\n plt.imshow(image)\n plt.axis(\"off\")\n plt.subplot(4, 4, 2 * i + 2)\n y = np.arange(top_probs.shape[-1])\n plt.grid()\n plt.barh(y, top_probs[i])\n plt.gca().invert_yaxis()\n plt.gca().set_axisbelow(True)\n plt.yticks(y, [cifar100.classes[index] for index in top_labels[i].numpy()])\n plt.xlabel(\"probability\")\nplt.subplots_adjust(wspace=0.5)\nplt.show()"
}
]
}

90
docs/doc/c84535bc-0d87-4786-a1fb-a9ea784b297f.json Normal file
View File

@ -0,0 +1,90 @@
{
"summary": "A summary of the comments discusses implementing advanced models with deep learning and attention mechanisms, using CLIP models for Convolutional Neural Networks and VisionTransformers, and initializing, converting, and loading state dicts into a CLIP model for evaluation.",
"details": [
{
"comment": "Class Bottleneck is defined as a subclass of nn.Module for residual block in a Convolutional Neural Network (CNN) architecture. It performs multiple convolutions, batch normalization, and activation functions. If stride > 1, it also includes an average pooling layer. The downsample parameter is set to None here but can be used if input and output planes are different.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":0-33",
"content": "from collections import OrderedDict\nfrom typing import Tuple, Union\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nclass Bottleneck(nn.Module):\n expansion = 4\n def __init__(self, inplanes, planes, stride=1):\n super().__init__()\n # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1\n self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)\n self.bn1 = nn.BatchNorm2d(planes)\n self.relu1 = nn.ReLU(inplace=True)\n self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(planes)\n self.relu2 = nn.ReLU(inplace=True)\n self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()\n self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)\n self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n self.relu3 = nn.ReLU(inplace=True)\n self.downsample = None\n self.stride = stride\n if stride > 1 or inplanes != planes * Bottleneck.expansion:"
},
{
"comment": "This code defines a convolutional block with downsampling and an AttentionPool2d module. The convolutional block performs convolutions with batch normalization and ReLU activations, while also allowing for optional downsampling through the defined `downsample` layer. The AttentionPool2d module is responsible for processing spatial features of input data using attention mechanism.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":34-60",
"content": " # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1\n self.downsample = nn.Sequential(OrderedDict([\n (\"-1\", nn.AvgPool2d(stride)),\n (\"0\", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),\n (\"1\", nn.BatchNorm2d(planes * self.expansion))\n ]))\n def forward(self, x: torch.Tensor):\n identity = x\n out = self.relu1(self.bn1(self.conv1(x)))\n out = self.relu2(self.bn2(self.conv2(out)))\n out = self.avgpool(out)\n out = self.bn3(self.conv3(out))\n if self.downsample is not None:\n identity = self.downsample(x)\n out += identity\n out = self.relu3(out)\n return out\nclass AttentionPool2d(nn.Module):\n def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):\n super().__init__()\n self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)"
},
{
"comment": "The code defines a class with `forward` method and initializes the necessary linear layers (`k_proj`, `q_proj`, `v_proj`, `c_proj`) for multi-head attention. It then processes input `x` by flattening, concatenating, adding positional embeddings, and finally calling `F.multi_head_attention_forward` with appropriate arguments.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":61-82",
"content": " self.k_proj = nn.Linear(embed_dim, embed_dim)\n self.q_proj = nn.Linear(embed_dim, embed_dim)\n self.v_proj = nn.Linear(embed_dim, embed_dim)\n self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)\n self.num_heads = num_heads\n def forward(self, x):\n x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC\n x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC\n x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC\n x, _ = F.multi_head_attention_forward(\n query=x[:1], key=x, value=x,\n embed_dim_to_check=x.shape[-1],\n num_heads=self.num_heads,\n q_proj_weight=self.q_proj.weight,\n k_proj_weight=self.k_proj.weight,\n v_proj_weight=self.v_proj.weight,\n in_proj_weight=None,\n in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),\n bias_k=None,\n bias_v=None,\n add_zero_attn=False,"
},
{
"comment": "Code snippet initializes a Conv2d layer, followed by BatchNorm2d layer for the stem of the modified ResNet. The stem consists of 3 convolution layers, each with stride 2 and padding 1. The BatchNorm2d layer normalizes the output of the convolution layer.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":83-108",
"content": " dropout_p=0,\n out_proj_weight=self.c_proj.weight,\n out_proj_bias=self.c_proj.bias,\n use_separate_proj_weight=True,\n training=self.training,\n need_weights=False\n )\n return x.squeeze(0)\nclass ModifiedResNet(nn.Module):\n \"\"\"\n A ResNet class that is similar to torchvision's but contains the following changes:\n - There are now 3 \"stem\" convolutions as opposed to 1, with an average pool instead of a max pool.\n - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1\n - The final pooling layer is a QKV attention instead of an average pool\n \"\"\"\n def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):\n super().__init__()\n self.output_dim = output_dim\n self.input_resolution = input_resolution\n # the 3-layer stem\n self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)\n self.bn1 = nn.BatchNorm2d(width // 2)"
},
{
"comment": "Code is defining a ResNet model with various layers such as convolution, batch normalization, ReLU activation, average pooling, and residual layers. It also includes an attention pooling layer. The ResNet model's feature dimension is set to 32.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":109-128",
"content": " self.relu1 = nn.ReLU(inplace=True)\n self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(width // 2)\n self.relu2 = nn.ReLU(inplace=True)\n self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)\n self.bn3 = nn.BatchNorm2d(width)\n self.relu3 = nn.ReLU(inplace=True)\n self.avgpool = nn.AvgPool2d(2)\n # residual layers\n self._inplanes = width # this is a *mutable* variable used during construction\n self.layer1 = self._make_layer(width, layers[0])\n self.layer2 = self._make_layer(width * 2, layers[1], stride=2)\n self.layer3 = self._make_layer(width * 4, layers[2], stride=2)\n self.layer4 = self._make_layer(width * 8, layers[3], stride=2)\n embed_dim = width * 32 # the ResNet feature dimension\n self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)\n def _make_layer(self, planes, blocks, stride=1):"
},
{
"comment": "129-138: Initialize layers with a Bottleneck block.\n140-146: Update inplanes for subsequent blocks.\n147-152: Append additional Bottleneck blocks to layers list.\n153-159: Return a nn.Sequential model from the layers list.\n160-166: Implement forward pass of the model, including stem and layer blocks.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":129-166",
"content": " layers = [Bottleneck(self._inplanes, planes, stride)]\n self._inplanes = planes * Bottleneck.expansion\n for _ in range(1, blocks):\n layers.append(Bottleneck(self._inplanes, planes))\n return nn.Sequential(*layers)\n def forward(self, x):\n def stem(x):\n x = self.relu1(self.bn1(self.conv1(x)))\n x = self.relu2(self.bn2(self.conv2(x)))\n x = self.relu3(self.bn3(self.conv3(x)))\n x = self.avgpool(x)\n return x\n x = x.type(self.conv1.weight.dtype)\n x = stem(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n x = self.attnpool(x)\n return x\nclass LayerNorm(nn.LayerNorm):\n \"\"\"Subclass torch's LayerNorm to handle fp16.\"\"\"\n def forward(self, x: torch.Tensor):\n orig_type = x.dtype\n ret = super().forward(x.type(torch.float32))\n return ret.type(orig_type)\nclass QuickGELU(nn.Module):\n def forward(self, x: torch.Tensor):"
},
{
"comment": "This code defines a Transformer model, specifically the Residual Attention Block and the main Transformer class. The ResidualAttentionBlock contains a MultiheadAttention layer, LayerNorm layers, and a feed-forward network. The Transformer class is initialized with width (d_model), number of layers, and number of heads for attention mechanism. It also accepts an optional attn_mask tensor.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":167-195",
"content": " return x * torch.sigmoid(1.702 * x)\nclass ResidualAttentionBlock(nn.Module):\n def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None):\n super().__init__()\n self.attn = nn.MultiheadAttention(d_model, n_head)\n self.ln_1 = LayerNorm(d_model)\n self.mlp = nn.Sequential(OrderedDict([\n (\"c_fc\", nn.Linear(d_model, d_model * 4)),\n (\"gelu\", QuickGELU()),\n (\"c_proj\", nn.Linear(d_model * 4, d_model))\n ]))\n self.ln_2 = LayerNorm(d_model)\n self.attn_mask = attn_mask\n def attention(self, x: torch.Tensor):\n self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None\n return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]\n def forward(self, x: torch.Tensor):\n x = x + self.attention(self.ln_1(x))\n x = x + self.mlp(self.ln_2(x))\n return x\nclass Transformer(nn.Module):\n def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None):"
},
{
"comment": "This code defines a VisionTransformer model with an input resolution, patch size, width, layers, number of heads, and output dimension. It initializes the model's parameters and contains forward pass and transformer class definitions.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":196-219",
"content": " super().__init__()\n self.width = width\n self.layers = layers\n self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)])\n def forward(self, x: torch.Tensor):\n return self.resblocks(x)\nclass VisionTransformer(nn.Module):\n def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int):\n super().__init__()\n self.input_resolution = input_resolution\n self.output_dim = output_dim\n self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)\n scale = width ** -0.5\n self.class_embedding = nn.Parameter(scale * torch.randn(width))\n self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))\n self.ln_pre = LayerNorm(width)\n self.transformer = Transformer(width, layers, heads)\n self.ln_post = LayerNorm(width)"
},
{
"comment": "This code defines a CLIP model, which consists of a convolutional layer followed by a Transformer. It performs feature extraction from an input image and then processes the features with a transformer network. The proj parameter is used for applying final linear projection if not None.\nCode location: \"clip/model.py\":249-271",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":220-247",
"content": " self.proj = nn.Parameter(scale * torch.randn(width, output_dim))\n def forward(self, x: torch.Tensor):\n x = self.conv1(x) # shape = [*, width, grid, grid]\n x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2]\n x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]\n x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width]\n x = x + self.positional_embedding.to(x.dtype)\n x = self.ln_pre(x)\n x = x.permute(1, 0, 2) # NLD -> LND\n x = self.transformer(x)\n x = x.permute(1, 0, 2) # LND -> NLD\n x = self.ln_post(x[:, 0, :])\n if self.proj is not None:\n x = x @ self.proj\n return x\nclass CLIP(nn.Module):\n def __init__(self,\n embed_dim: int,\n # vision\n image_resolution: int,\n vision_layers: Union[Tuple[int, int, int, int], int],"
},
{
"comment": "Initializing a model with provided parameters for vision and language processing.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":248-277",
"content": " vision_width: int,\n vision_patch_size: int,\n # text\n context_length: int,\n vocab_size: int,\n transformer_width: int,\n transformer_heads: int,\n transformer_layers: int\n ):\n super().__init__()\n self.context_length = context_length\n if isinstance(vision_layers, (tuple, list)):\n vision_heads = vision_width * 32 // 64\n self.visual = ModifiedResNet(\n layers=vision_layers,\n output_dim=embed_dim,\n heads=vision_heads,\n input_resolution=image_resolution,\n width=vision_width\n )\n else:\n vision_heads = vision_width // 64\n self.visual = VisionTransformer(\n input_resolution=image_resolution,\n patch_size=vision_patch_size,\n width=vision_width,\n layers=vision_layers,\n heads=vision_heads,"
},
{
"comment": "This code initializes the model's parameters. It sets up layers such as transformer, token embedding, positional embedding, layer normalization, and logit scale. The initialize_parameters method is used to set up initial values for the embeddings with small standard deviations.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":278-304",
"content": " output_dim=embed_dim\n )\n self.transformer = Transformer(\n width=transformer_width,\n layers=transformer_layers,\n heads=transformer_heads,\n attn_mask=self.build_attention_mask()\n )\n self.vocab_size = vocab_size\n self.token_embedding = nn.Embedding(vocab_size, transformer_width)\n self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))\n self.ln_final = LayerNorm(transformer_width)\n self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))\n self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))\n self.initialize_parameters()\n def initialize_parameters(self):\n nn.init.normal_(self.token_embedding.weight, std=0.02)\n nn.init.normal_(self.positional_embedding, std=0.01)\n if isinstance(self.visual, ModifiedResNet):\n if self.visual.attnpool is not None:\n std = self.visual.attnpool.c_proj.in_features ** -0.5"
},
{
"comment": "This code initializes the weights of various layers in a neural network model. It uses different initialization methods and standards deviations for different types of layers, such as normalizing the weights for attention pools, ResNet blocks, and feedforward layers.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":305-321",
"content": " nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)\n nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)\n nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)\n nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)\n for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:\n for name, param in resnet_block.named_parameters():\n if name.endswith(\"bn3.weight\"):\n nn.init.zeros_(param)\n proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)\n attn_std = self.transformer.width ** -0.5\n fc_std = (2 * self.transformer.width) ** -0.5\n for block in self.transformer.resblocks:\n nn.init.normal_(block.attn.in_proj_weight, std=attn_std)\n nn.init.normal_(block.attn.out_proj.weight, std=proj_std)\n nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)"
},
{
"comment": "1. Initializes the model parameters with normal distribution.\n2. Builds a causal attention mask for the transformer, filling with -inf for lower diagonal elements.\n3. Encodes image using the provided visual encoder.\n4. Encodes text using token embedding and positional embedding followed by the transformer.\n5. Permutes the output to ensure it's in NLD format (batch_size, sequence_length, feature_dimensions).",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":322-348",
"content": " nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)\n if self.text_projection is not None:\n nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)\n def build_attention_mask(self):\n # lazily create causal attention mask, with full attention between the vision tokens\n # pytorch uses additive attention mask; fill with -inf\n mask = torch.empty(self.context_length, self.context_length)\n mask.fill_(float(\"-inf\"))\n mask.triu_(1) # zero out the lower diagonal\n return mask\n @property\n def dtype(self):\n return self.visual.conv1.weight.dtype\n def encode_image(self, image):\n return self.visual(image.type(self.dtype))\n def encode_text(self, text):\n x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model]\n x = x + self.positional_embedding.type(self.dtype)\n x = x.permute(1, 0, 2) # NLD -> LND\n x = self.transformer(x)\n x = x.permute(1, 0, 2) # LND -> NLD"
},
{
"comment": "\"clip/model.py\":349-375",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":349-375",
"content": " x = self.ln_final(x).type(self.dtype)\n # x.shape = [batch_size, n_ctx, transformer.width]\n # take features from the eot embedding (eot_token is the highest number in each sequence)\n x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection\n return x\n def forward(self, image, text):\n image_features = self.encode_image(image)\n text_features = self.encode_text(text)\n # normalized features\n image_features = image_features / image_features.norm(dim=1, keepdim=True)\n text_features = text_features / text_features.norm(dim=1, keepdim=True)\n # cosine similarity as logits\n logit_scale = self.logit_scale.exp()\n logits_per_image = logit_scale * image_features @ text_features.t()\n logits_per_text = logits_per_image.t()\n # shape = [global_batch_size, global_batch_size]\n return logits_per_image, logits_per_text\ndef convert_weights(model: nn.Module):\n \"\"\"Convert applicable model parameters to fp16\"\"\""
},
{
"comment": "This code applies a function to convert weights of certain layers (Conv1d, Conv2d, Linear, MultiheadAttention) from float32 to float16. It also builds a model using a state dictionary.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":377-403",
"content": " def _convert_weights_to_fp16(l):\n if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):\n l.weight.data = l.weight.data.half()\n if l.bias is not None:\n l.bias.data = l.bias.data.half()\n if isinstance(l, nn.MultiheadAttention):\n for attr in [*[f\"{s}_proj_weight\" for s in [\"in\", \"q\", \"k\", \"v\"]], \"in_proj_bias\", \"bias_k\", \"bias_v\"]:\n tensor = getattr(l, attr)\n if tensor is not None:\n tensor.data = tensor.data.half()\n for name in [\"text_projection\", \"proj\"]:\n if hasattr(l, name):\n attr = getattr(l, name)\n if attr is not None:\n attr.data = attr.data.half()\n model.apply(_convert_weights_to_fp16)\ndef build_model(state_dict: dict):\n vit = \"visual.proj\" in state_dict\n if vit:\n vision_width = state_dict[\"visual.conv1.weight\"].shape[0]\n vision_layers = len([k for k in state_dict.keys() if k.startswith(\"visual.\") and k.endswith(\".attn.in_proj_weight\")])"
},
{
"comment": "Determine the vision layers' count, widths, and image resolution.\n- Determines how many vision layers exist for each layer number (1, 2, 3, 4) by counting unique keys with matching prefixes in the state dictionary.\n- If the \"visual.attnpool\" key exists, calculates the number of patches along one dimension based on the positional embedding shape and sets vision_patch_size to None. Asserts that the shape matches a specific condition.\n- Computes the image resolution by multiplying vision_patch_size with grid size (rounded down integer value of square root of positional embedding's shape[0] minus one).\n- If no \"visual.attnpool\" key exists, calculates the vision layer count and width based on keys matching prefixes in the state dictionary. Calculates output_width similarly to grid size calculation above but for the attention pooling case.\n- Sets vision_patch_size to None since it's not available from the state dictionary.\n- Finally, calculates image resolution by multiplying output_width with a fixed value (32).\n- Determines embed_dim, context_length and vocab_size based on matching keys in the state dictionary.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":404-420",
"content": " vision_patch_size = state_dict[\"visual.conv1.weight\"].shape[-1]\n grid_size = round((state_dict[\"visual.positional_embedding\"].shape[0] - 1) ** 0.5)\n image_resolution = vision_patch_size * grid_size\n else:\n counts: list = [len(set(k.split(\".\")[2] for k in state_dict if k.startswith(f\"visual.layer{b}\"))) for b in [1, 2, 3, 4]]\n vision_layers = tuple(counts)\n vision_width = state_dict[\"visual.layer1.0.conv1.weight\"].shape[0]\n output_width = round((state_dict[\"visual.attnpool.positional_embedding\"].shape[0] - 1) ** 0.5)\n vision_patch_size = None\n assert output_width ** 2 + 1 == state_dict[\"visual.attnpool.positional_embedding\"].shape[0]\n image_resolution = output_width * 32\n embed_dim = state_dict[\"text_projection\"].shape[1]\n context_length = state_dict[\"positional_embedding\"].shape[0]\n vocab_size = state_dict[\"token_embedding.weight\"].shape[0]\n transformer_width = state_dict[\"ln_final.weight\"].shape[0]\n transformer_heads = transformer_width // 64"
},
{
"comment": "This code initializes a CLIP model with given dimensions and layers, removes unnecessary state dict keys, converts weights, and loads the modified state dict into the model for evaluation.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/clip/model.py\":421-435",
"content": " transformer_layers = len(set(k.split(\".\")[2] for k in state_dict if k.startswith(\"transformer.resblocks\")))\n model = CLIP(\n embed_dim,\n image_resolution, vision_layers, vision_width, vision_patch_size,\n context_length, vocab_size, transformer_width, transformer_heads, transformer_layers\n )\n for key in [\"input_resolution\", \"context_length\", \"vocab_size\"]:\n if key in state_dict:\n del state_dict[key]\n convert_weights(model)\n model.load_state_dict(state_dict)\n return model.eval()"
}
]
}

15
docs/doc/dae87818-0507-4387-9a64-47d75db3f8a5.json Normal file
View File

@ -0,0 +1,15 @@
{
"summary": "This code defines functions for creating entry points to load CLIP models and converting PIL images into tensors, while also mapping model names and updating the global namespace with different model entrypoints.",
"details": [
{
"comment": "This code defines a function _create_hub_entrypoint that creates an entry point for loading CLIP models. It also imports necessary dependencies and maps available model names to remove any special characters for compatibility.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/hubconf.py\":0-31",
"content": "from clip.clip import tokenize as _tokenize, load as _load, available_models as _available_models\nimport re\nimport string\ndependencies = [\"torch\", \"torchvision\", \"ftfy\", \"regex\", \"tqdm\"]\n# For compatibility (cannot include special characters in function name)\nmodel_functions = { model: re.sub(f'[{string.punctuation}]', '_', model) for model in _available_models()}\ndef _create_hub_entrypoint(model):\n def entrypoint(**kwargs): \n return _load(model, **kwargs)\n entrypoint.__doc__ = f\"\"\"Loads the {model} CLIP model\n Parameters\n ----------\n device : Union[str, torch.device]\n The device to put the loaded model\n jit : bool\n Whether to load the optimized JIT model or more hackable non-JIT model (default).\n download_root: str\n path to download the model files; by default, it uses \"~/.cache/clip\"\n Returns\n -------\n model : torch.nn.Module\n The {model} CLIP model\n preprocess : Callable[[PIL.Image], torch.Tensor]"
},
{
"comment": "This code defines a function that converts a PIL image into a tensor. It also creates entrypoints for different models using _available_models() and updates the global namespace with these entrypoints.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/hubconf.py\":32-41",
"content": " A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input\n \"\"\"\n return entrypoint\ndef tokenize():\n return _tokenize\n_entrypoints = {model_functions[model]: _create_hub_entrypoint(model) for model in _available_models()}\nglobals().update(_entrypoints)"
}
]
}

45
docs/doc/e03e2699-e000-420e-a96a-a8c8714019d3.json Normal file
View File

@ -0,0 +1,45 @@
{
"summary": "OpenAI's CLIP model is a multimodal AI for computer vision and zero-shot image classification. It uses ResNet50 or Vision Transformer as encoders but has limitations like dataset building, performance variability, and potential biases. Training data includes website crawling and YFCC100M datasets. The code provides a Google Form link for feedback on model performance and risks.",
"details": [
{
"comment": "Storage location: \"model-card.md\":0-14\nCode description: This code is a model card for CLIP, a multimodal model developed by OpenAI researchers. The model aims to understand what contributes to robustness in computer vision tasks and test generalization abilities in zero-shot image classification tasks. It was not designed for general deployment and requires careful study before being used in specific contexts. The model card provides details on the development date, model type (ResNet50 with modifications as an image encoder and a masked self-attention Transformer as a text encoder), and that the encoders are trained to maximize similarity of inputs.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":0-14",
"content": "# Model Card: CLIP\nInspired by [Model Cards for Model Reporting (Mitchell et al.)](https://arxiv.org/abs/1810.03993) and [Lessons from Archives (Jo & Gebru)](https://arxiv.org/pdf/1912.10389.pdf), we\u2019re providing some accompanying information about the multimodal model.\n## Model Details\nThe CLIP model was developed by researchers at OpenAI to learn about what contributes to robustness in computer vision tasks. The model was also developed to test the ability of models to generalize to arbitrary image classification tasks in a zero-shot manner. It was not developed for general model deployment - to deploy models like CLIP, researchers will first need to carefully study their capabilities in relation to the specific context they\u2019re being deployed within.\n### Model Date\nJanuary 2021\n### Model Type\nThe base model uses a ResNet50 with several modifications as an image encoder and uses a masked self-attention Transformer as a text encoder. These encoders are trained to maximize the similarity of (i"
},
{
"comment": "This code describes the CLIP model, a contrastive image-text model with variants using Vision Transformer or ResNet image encoder. It mentions the different released versions of the model and provides links to relevant documents such as the blog post and paper for further details on specifications and intended use.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":14-35",
"content": "mage, text) pairs via a contrastive loss. There is also a variant of the model where the ResNet image encoder is replaced with a Vision Transformer.\n### Model Versions\nInitially, we\u2019ve released one CLIP model based on the Vision Transformer architecture equivalent to ViT-B/32, along with the RN50 model, using the architecture equivalent to ResNet-50.\nAs part of the staged release process, we have also released the RN101 model, as well as RN50x4, a RN50 scaled up 4x according to the [EfficientNet](https://arxiv.org/abs/1905.11946) scaling rule. In July 2021, we additionally released the RN50x16 and ViT-B/16 models, and in January 2022, the RN50x64 and ViT-L/14 models were released. Lastly, the ViT-L/14@336px model was released in April 2022.\nPlease see the paper linked below for further details about their specification.\n### Documents\n- [Blog Post](https://openai.com/blog/clip/)\n- [CLIP Paper](https://arxiv.org/abs/2103.00020)\n## Model Use\n### Intended Use\nThe model is intended as a research outp"
},
{
"comment": "This code snippet provides information about the intended use and out-of-scope use cases for a specific model. It explains that the primary audience is AI researchers who will use it to study various aspects of computer vision models, such as robustness, generalization, capabilities, biases, and constraints. Deployed use cases are currently out of scope, while non-deployed use cases should only be considered after thorough in-domain testing with a fixed class taxonomy.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":35-45",
"content": "ut for research communities. We hope that this model will enable researchers to better understand and explore zero-shot, arbitrary image classification. We also hope it can be used for interdisciplinary studies of the potential impact of such models - the CLIP paper includes a discussion of potential downstream impacts to provide an example for this sort of analysis.\n#### Primary intended uses\nThe primary intended users of these models are AI researchers.\nWe primarily imagine the model will be used by researchers to better understand robustness, generalization, and other capabilities, biases, and constraints of computer vision models.\n### Out-of-Scope Use Cases\n**Any** deployed use case of the model - whether commercial or not - is currently out of scope. Non-deployed use cases such as image search in a constrained environment, are also not recommended unless there is thorough in-domain testing of the model with a specific, fixed class taxonomy. This is because our safety assessment demonst"
},
{
"comment": "The code highlights the need for task-specific testing due to CLIP's performance variability and cautions against unconstrained deployment in certain use cases. It also emphasizes the model's English language limitations and provides information on the data used for training, including crawling websites and using pre-existing datasets like YFCC100M.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":45-55",
"content": "rated a high need for task specific testing especially given the variability of CLIP\u2019s performance with different class taxonomies. This makes untested and unconstrained deployment of the model in any use case currently potentially harmful. \nCertain use cases which would fall under the domain of surveillance and facial recognition are always out-of-scope regardless of performance of the model. This is because the use of artificial intelligence for tasks such as these can be premature currently given the lack of testing norms and checks to ensure its fair use.\nSince the model has not been purposefully trained in or evaluated on any languages other than English, its use should be limited to English language use cases.\n## Data\nThe model was trained on publicly available image-caption data. This was done through a combination of crawling a handful of websites and using commonly-used pre-existing image datasets such as [YFCC100M](http://projects.dfki.uni-kl.de/yfcc100m/). A large portion of the"
},
{
"comment": "The code is describing the data used in building a dataset, its mission statement, and discussing performance and limitations. The data comes from internet crawling, mainly focusing on more developed nations and younger male users. The goal was to test robustness and generalizability in computer vision tasks. The dataset will not be released for commercial or deployed use. Performance is evaluated across various benchmarks and computer vision datasets like OCR to text.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":55-67",
"content": " data comes from our crawling of the internet. This means that the data is more representative of people and societies most connected to the internet which tend to skew towards more developed nations, and younger, male users.\n### Data Mission Statement\nOur goal with building this dataset was to test out robustness and generalizability in computer vision tasks. As a result, the focus was on gathering large quantities of data from different publicly-available internet data sources. The data was gathered in a mostly non-interventionist manner. However, we only crawled websites that had policies against excessively violent and adult images and allowed us to filter out such content. We do not intend for this dataset to be used as the basis for any commercial or deployed model and will not be releasing the dataset.\n## Performance and Limitations\n### Performance\nWe have evaluated the performance of CLIP on a wide range of benchmarks across a variety of computer vision datasets such as OCR to textu"
},
{
"comment": "The code lists various datasets used in the evaluation of the model's performance.\n\nIt highlights that CLIP has limitations, such as difficulties with fine-grained classification and counting objects. It also addresses issues related to fairness and bias, while noting a limitation in their approach by using linear probes for evaluation, which may underestimate model performance.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":67-105",
"content": "re recognition to fine-grained classification. The paper describes model performance on the following datasets:\n- Food101\n- CIFAR10 \n- CIFAR100 \n- Birdsnap\n- SUN397\n- Stanford Cars\n- FGVC Aircraft\n- VOC2007\n- DTD\n- Oxford-IIIT Pet dataset\n- Caltech101\n- Flowers102\n- MNIST \n- SVHN \n- IIIT5K \n- Hateful Memes \n- SST-2\n- UCF101\n- Kinetics700\n- Country211\n- CLEVR Counting\n- KITTI Distance\n- STL-10\n- RareAct\n- Flickr30\n- MSCOCO\n- ImageNet\n- ImageNet-A\n- ImageNet-R\n- ImageNet Sketch\n- ObjectNet (ImageNet Overlap)\n- Youtube-BB\n- ImageNet-Vid\n## Limitations\nCLIP and our analysis of it have a number of limitations. CLIP currently struggles with respect to certain tasks such as fine grained classification and counting objects. CLIP also poses issues with regards to fairness and bias which we discuss in the paper and briefly in the next section. Additionally, our approach to testing CLIP also has an important limitation- in many cases we have used linear probes to evaluate the performance of CLIP and there is evidence suggesting that linear probes can underestimate model performance."
},
{
"comment": "Discusses the impact of class design on CLIP's biases, highlights disparities based on race and gender using Fairface dataset, and mentions accuracy over 96% for gender classification across all races.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":107-111",
"content": "### Bias and Fairness\nWe find that the performance of CLIP - and the specific biases it exhibits - can depend significantly on class design and the choices one makes for categories to include and exclude. We tested the risk of certain kinds of denigration with CLIP by classifying images of people from [Fairface](https://arxiv.org/abs/1908.04913) into crime-related and non-human animal categories. We found significant disparities with respect to race and gender. Additionally, we found that these disparities could shift based on how the classes were constructed. (Details captured in the Broader Impacts Section in the paper).\nWe also tested the performance of CLIP on gender, race and age classification using the Fairface dataset (We default to using race categories as they are constructed in the Fairface dataset.) in order to assess quality of performance across different demographics. We found accuracy >96% across all races for gender classification with \u2018Middle Eastern\u2019 having the highest"
},
{
"comment": "This code is providing the accuracy of the model for various classifications and emphasizing that these evaluations are to test performance and identify potential risks, not to endorse such tasks. It also provides a link to a Google Form for questions or comments about the model.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/model-card.md\":111-119",
"content": " accuracy (98.4%) and \u2018White\u2019 having the lowest (96.5%). Additionally, CLIP averaged ~93% for racial classification and ~63% for age classification. Our use of evaluations to test for gender, race and age classification as well as denigration harms is simply to evaluate performance of the model across people and surface potential risks and not to demonstrate an endorsement/enthusiasm for such tasks.\n## Feedback\n### Where to send questions or comments about the model\nPlease use [this Google Form](https://forms.gle/Uv7afRH5dvY34ZEs9)"
}
]
}

10
docs/doc/feecb0f0-fc6b-440d-8a81-d79701c81913.json Normal file
View File

@ -0,0 +1,10 @@
{
"summary": "This code provides instructions to download and extract the Country211 dataset, a geolocation image classification dataset created from YFCC100m. The dataset contains balanced samples for training, validation, and testing for each country with corresponding ISO-3166 codes.",
"details": [
{
"comment": "This code provides instructions to download and extract the Country211 dataset, a geolocation image classification dataset created from YFCC100m. The dataset contains balanced samples for training, validation, and testing for each country with corresponding ISO-3166 codes.",
"location": "\"/media/root/Toshiba XG3/works/CLIP/docs/src/data/country211.md\":0-11",
"content": "# The Country211 Dataset\nIn the paper, we used an image classification dataset called Country211, to evaluate the model's capability on geolocation. To do so, we filtered the YFCC100m dataset that have GPS coordinate corresponding to a [ISO-3166 country code](https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes) and created a balanced dataset by sampling 150 train images, 50 validation images, and 100 test images images for each country.\nThe following command will download an 11GB archive countaining the images and extract into a subdirectory `country211`:\n```bash\nwget https://openaipublic.azureedge.net/clip/data/country211.tgz\ntar zxvf country211.tgz\n```\nThese images are a subset of the YFCC100m dataset. Use of the underlying media files is subject to the Creative Commons licenses chosen by their creators/uploaders. For more information about the YFCC100M dataset, visit [the official website](https://multimediacommons.wordpress.com/yfcc100m-core-dataset/)."
}
]
}

724
docs/index.html Normal file
View File

@ -0,0 +1,724 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="icon"
href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' fill='currentColor' class='bi bi-search' viewBox='0 0 16 16'%3E%3Cpath d='M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0'/%3E%3C/svg%3E"
type="image/svg+xml">
<title>Search Code By Comment</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css">
<script type="text/javascript">
/**
* FlexSearch.js v0.7.31 (Bundle)
* Author and Copyright: Thomas Wilkerling
* Licence: Apache-2.0
* Hosted by Nextapps GmbH
* https://github.com/nextapps-de/flexsearch
*/
(function _f(self) {
'use strict'; try { if (module) self = module } catch (e) { } self._factory = _f; var t; function u(a) { return "undefined" !== typeof a ? a : !0 } function aa(a) { const b = Array(a); for (let c = 0; c < a; c++)b[c] = v(); return b } function v() { return Object.create(null) } function ba(a, b) { return b.length - a.length } function x(a) { return "string" === typeof a } function C(a) { return "object" === typeof a } function D(a) { return "function" === typeof a }; function ca(a, b) { var c = da; if (a && (b && (a = E(a, b)), this.H && (a = E(a, this.H)), this.J && 1 < a.length && (a = E(a, this.J)), c || "" === c)) { a = a.split(c); if (this.filter) { b = this.filter; c = a.length; const d = []; for (let e = 0, f = 0; e < c; e++) { const g = a[e]; g && !b[g] && (d[f++] = g) } a = d } return a } return a } const da = /[\s\xA0\u2000-\u200B\u2028\u2029\u3000\ufeff!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/
/* /[\p{Z}\p{S}\p{P}\p{C}]+/u */, ea = /[\u0300-\u036f]/g;
function fa(a, b) { const c = Object.keys(a), d = c.length, e = []; let f = "", g = 0; for (let h = 0, k, m; h < d; h++)k = c[h], (m = a[k]) ? (e[g++] = F(b ? "(?!\\b)" + k + "(\\b|_)" : k), e[g++] = m) : f += (f ? "|" : "") + k; f && (e[g++] = F(b ? "(?!\\b)(" + f + ")(\\b|_)" : "(" + f + ")"), e[g] = ""); return e } function E(a, b) { for (let c = 0, d = b.length; c < d && (a = a.replace(b[c], b[c + 1]), a); c += 2); return a } function F(a) { return new RegExp(a, "g") } function ha(a) { let b = "", c = ""; for (let d = 0, e = a.length, f; d < e; d++)(f = a[d]) !== c && (b += c = f); return b }; var ja = { encode: ia, F: !1, G: "" }; function ia(a) { return ca.call(this, ("" + a).toLowerCase(), !1) }; const ka = {}, G = {}; function la(a) { I(a, "add"); I(a, "append"); I(a, "search"); I(a, "update"); I(a, "remove") } function I(a, b) { a[b + "Async"] = function () { const c = this, d = arguments; var e = d[d.length - 1]; let f; D(e) && (f = e, delete d[d.length - 1]); e = new Promise(function (g) { setTimeout(function () { c.async = !0; const h = c[b].apply(c, d); c.async = !1; g(h) }) }); return f ? (e.then(f), this) : e } }; function ma(a, b, c, d) {
const e = a.length; let f = [], g, h, k = 0; d && (d = []); for (let m = e - 1; 0 <= m; m--) { const n = a[m], w = n.length, q = v(); let r = !g; for (let l = 0; l < w; l++) { const p = n[l], z = p.length; if (z) for (let B = 0, A, y; B < z; B++)if (y = p[B], g) { if (g[y]) { if (!m) if (c) c--; else if (f[k++] = y, k === b) return f; if (m || d) q[y] = 1; r = !0 } if (d && (A = (h[y] || 0) + 1, h[y] = A, A < e)) { const H = d[A - 2] || (d[A - 2] = []); H[H.length] = y } } else q[y] = 1 } if (d) g || (h = q); else if (!r) return []; g = q } if (d) for (let m = d.length - 1, n, w; 0 <= m; m--) {
n = d[m]; w = n.length; for (let q = 0, r; q < w; q++)if (r =
n[q], !g[r]) { if (c) c--; else if (f[k++] = r, k === b) return f; g[r] = 1 }
} return f
} function na(a, b) { const c = v(), d = v(), e = []; for (let f = 0; f < a.length; f++)c[a[f]] = 1; for (let f = 0, g; f < b.length; f++) { g = b[f]; for (let h = 0, k; h < g.length; h++)k = g[h], c[k] && !d[k] && (d[k] = 1, e[e.length] = k) } return e }; function J(a) { this.l = !0 !== a && a; this.cache = v(); this.h = [] } function oa(a, b, c) { C(a) && (a = a.query); let d = this.cache.get(a); d || (d = this.search(a, b, c), this.cache.set(a, d)); return d } J.prototype.set = function (a, b) { if (!this.cache[a]) { var c = this.h.length; c === this.l ? delete this.cache[this.h[c - 1]] : c++; for (--c; 0 < c; c--)this.h[c] = this.h[c - 1]; this.h[0] = a } this.cache[a] = b }; J.prototype.get = function (a) { const b = this.cache[a]; if (this.l && b && (a = this.h.indexOf(a))) { const c = this.h[a - 1]; this.h[a - 1] = this.h[a]; this.h[a] = c } return b }; const qa = { memory: { charset: "latin:extra", D: 3, B: 4, m: !1 }, performance: { D: 3, B: 3, s: !1, context: { depth: 2, D: 1 } }, match: { charset: "latin:extra", G: "reverse" }, score: { charset: "latin:advanced", D: 20, B: 3, context: { depth: 3, D: 9 } }, "default": {} }; function ra(a, b, c, d, e, f, g) { setTimeout(function () { const h = a(c ? c + "." + d : d, JSON.stringify(g)); h && h.then ? h.then(function () { b.export(a, b, c, e, f + 1) }) : b.export(a, b, c, e, f + 1) }) }; function K(a, b) {
if (!(this instanceof K)) return new K(a); var c; if (a) { x(a) ? a = qa[a] : (c = a.preset) && (a = Object.assign({}, c[c], a)); c = a.charset; var d = a.lang; x(c) && (-1 === c.indexOf(":") && (c += ":default"), c = G[c]); x(d) && (d = ka[d]) } else a = {}; let e, f, g = a.context || {}; this.encode = a.encode || c && c.encode || ia; this.register = b || v(); this.D = e = a.resolution || 9; this.G = b = c && c.G || a.tokenize || "strict"; this.depth = "strict" === b && g.depth; this.l = u(g.bidirectional); this.s = f = u(a.optimize); this.m = u(a.fastupdate); this.B = a.minlength || 1; this.C =
a.boost; this.map = f ? aa(e) : v(); this.A = e = g.resolution || 1; this.h = f ? aa(e) : v(); this.F = c && c.F || a.rtl; this.H = (b = a.matcher || d && d.H) && fa(b, !1); this.J = (b = a.stemmer || d && d.J) && fa(b, !0); if (c = b = a.filter || d && d.filter) { c = b; d = v(); for (let h = 0, k = c.length; h < k; h++)d[c[h]] = 1; c = d } this.filter = c; this.cache = (b = a.cache) && new J(b)
} t = K.prototype; t.append = function (a, b) { return this.add(a, b, !0) };
t.add = function (a, b, c, d) {
if (b && (a || 0 === a)) {
if (!d && !c && this.register[a]) return this.update(a, b); b = this.encode(b); if (d = b.length) {
const m = v(), n = v(), w = this.depth, q = this.D; for (let r = 0; r < d; r++) {
let l = b[this.F ? d - 1 - r : r]; var e = l.length; if (l && e >= this.B && (w || !n[l])) {
var f = L(q, d, r), g = ""; switch (this.G) {
case "full": if (2 < e) { for (f = 0; f < e; f++)for (var h = e; h > f; h--)if (h - f >= this.B) { var k = L(q, d, r, e, f); g = l.substring(f, h); M(this, n, g, k, a, c) } break } case "reverse": if (1 < e) {
for (h = e - 1; 0 < h; h--)g = l[h] + g, g.length >= this.B && M(this, n,
g, L(q, d, r, e, h), a, c); g = ""
} case "forward": if (1 < e) { for (h = 0; h < e; h++)g += l[h], g.length >= this.B && M(this, n, g, f, a, c); break } default: if (this.C && (f = Math.min(f / this.C(b, l, r) | 0, q - 1)), M(this, n, l, f, a, c), w && 1 < d && r < d - 1) for (e = v(), g = this.A, f = l, h = Math.min(w + 1, d - r), e[f] = 1, k = 1; k < h; k++)if ((l = b[this.F ? d - 1 - r - k : r + k]) && l.length >= this.B && !e[l]) { e[l] = 1; const p = this.l && l > f; M(this, m, p ? f : l, L(g + (d / 2 > g ? 0 : 1), d, r, h - 1, k - 1), a, c, p ? l : f) }
}
}
} this.m || (this.register[a] = 1)
}
} return this
};
function L(a, b, c, d, e) { return c && 1 < a ? b + (d || 0) <= a ? c + (e || 0) : (a - 1) / (b + (d || 0)) * (c + (e || 0)) + 1 | 0 : 0 } function M(a, b, c, d, e, f, g) { let h = g ? a.h : a.map; if (!b[c] || g && !b[c][g]) a.s && (h = h[d]), g ? (b = b[c] || (b[c] = v()), b[g] = 1, h = h[g] || (h[g] = v())) : b[c] = 1, h = h[c] || (h[c] = []), a.s || (h = h[d] || (h[d] = [])), f && h.includes(e) || (h[h.length] = e, a.m && (a = a.register[e] || (a.register[e] = []), a[a.length] = h)) }
t.search = function (a, b, c) {
c || (!b && C(a) ? (c = a, a = c.query) : C(b) && (c = b)); let d = [], e; let f, g = 0; if (c) { a = c.query || a; b = c.limit; g = c.offset || 0; var h = c.context; f = c.suggest } if (a && (a = this.encode("" + a), e = a.length, 1 < e)) { c = v(); var k = []; for (let n = 0, w = 0, q; n < e; n++)if ((q = a[n]) && q.length >= this.B && !c[q]) if (this.s || f || this.map[q]) k[w++] = q, c[q] = 1; else return d; a = k; e = a.length } if (!e) return d; b || (b = 100); h = this.depth && 1 < e && !1 !== h; c = 0; let m; h ? (m = a[0], c = 1) : 1 < e && a.sort(ba); for (let n, w; c < e; c++) {
w = a[c]; h ? (n = sa(this, d, f, b, g, 2 === e, w,
m), f && !1 === n && d.length || (m = w)) : n = sa(this, d, f, b, g, 1 === e, w); if (n) return n; if (f && c === e - 1) { k = d.length; if (!k) { if (h) { h = 0; c = -1; continue } return d } if (1 === k) return ta(d[0], b, g) }
} return ma(d, b, g, f)
};
function sa(a, b, c, d, e, f, g, h) { let k = [], m = h ? a.h : a.map; a.s || (m = ua(m, g, h, a.l)); if (m) { let n = 0; const w = Math.min(m.length, h ? a.A : a.D); for (let q = 0, r = 0, l, p; q < w; q++)if (l = m[q]) if (a.s && (l = ua(l, g, h, a.l)), e && l && f && (p = l.length, p <= e ? (e -= p, l = null) : (l = l.slice(e), e = 0)), l && (k[n++] = l, f && (r += l.length, r >= d))) break; if (n) { if (f) return ta(k, d, 0); b[b.length] = k; return } } return !c && k } function ta(a, b, c) { a = 1 === a.length ? a[0] : [].concat.apply([], a); return c || a.length > b ? a.slice(c, c + b) : a }
function ua(a, b, c, d) { c ? (d = d && b > c, a = (a = a[d ? b : c]) && a[d ? c : b]) : a = a[b]; return a } t.contain = function (a) { return !!this.register[a] }; t.update = function (a, b) { return this.remove(a).add(a, b) };
t.remove = function (a, b) { const c = this.register[a]; if (c) { if (this.m) for (let d = 0, e; d < c.length; d++)e = c[d], e.splice(e.indexOf(a), 1); else N(this.map, a, this.D, this.s), this.depth && N(this.h, a, this.A, this.s); b || delete this.register[a]; if (this.cache) { b = this.cache; for (let d = 0, e, f; d < b.h.length; d++)f = b.h[d], e = b.cache[f], e.includes(a) && (b.h.splice(d--, 1), delete b.cache[f]) } } return this };
function N(a, b, c, d, e) { let f = 0; if (a.constructor === Array) if (e) b = a.indexOf(b), -1 !== b ? 1 < a.length && (a.splice(b, 1), f++) : f++; else { e = Math.min(a.length, c); for (let g = 0, h; g < e; g++)if (h = a[g]) f = N(h, b, c, d, e), d || f || delete a[g] } else for (let g in a) (f = N(a[g], b, c, d, e)) || delete a[g]; return f } t.searchCache = oa;
t.export = function (a, b, c, d, e) { let f, g; switch (e || (e = 0)) { case 0: f = "reg"; if (this.m) { g = v(); for (let h in this.register) g[h] = 1 } else g = this.register; break; case 1: f = "cfg"; g = { doc: 0, opt: this.s ? 1 : 0 }; break; case 2: f = "map"; g = this.map; break; case 3: f = "ctx"; g = this.h; break; default: return }ra(a, b || this, c, f, d, e, g); return !0 }; t.import = function (a, b) { if (b) switch (x(b) && (b = JSON.parse(b)), a) { case "cfg": this.s = !!b.opt; break; case "reg": this.m = !1; this.register = b; break; case "map": this.map = b; break; case "ctx": this.h = b } }; la(K.prototype); function va(a) { a = a.data; var b = self._index; const c = a.args; var d = a.task; switch (d) { case "init": d = a.options || {}; a = a.factory; b = d.encode; d.cache = !1; b && 0 === b.indexOf("function") && (d.encode = Function("return " + b)()); a ? (Function("return " + a)()(self), self._index = new self.FlexSearch.Index(d), delete self.FlexSearch) : self._index = new K(d); break; default: a = a.id, b = b[d].apply(b, c), postMessage("search" === d ? { id: a, msg: b } : { id: a }) } }; let wa = 0; function O(a) { if (!(this instanceof O)) return new O(a); var b; a ? D(b = a.encode) && (a.encode = b.toString()) : a = {}; (b = (self || window)._factory) && (b = b.toString()); const c = "undefined" === typeof window && self.exports, d = this; this.o = xa(b, c, a.worker); this.h = v(); if (this.o) { if (c) this.o.on("message", function (e) { d.h[e.id](e.msg); delete d.h[e.id] }); else this.o.onmessage = function (e) { e = e.data; d.h[e.id](e.msg); delete d.h[e.id] }; this.o.postMessage({ task: "init", factory: b, options: a }) } } P("add"); P("append"); P("search");
P("update"); P("remove"); function P(a) { O.prototype[a] = O.prototype[a + "Async"] = function () { const b = this, c = [].slice.call(arguments); var d = c[c.length - 1]; let e; D(d) && (e = d, c.splice(c.length - 1, 1)); d = new Promise(function (f) { setTimeout(function () { b.h[++wa] = f; b.o.postMessage({ task: a, id: wa, args: c }) }) }); return e ? (d.then(e), this) : d } }
function xa(a, b, c) { let d; try { d = b ? eval('new (require("worker_threads")["Worker"])("../dist/node/node.js")') : a ? new Worker(URL.createObjectURL(new Blob(["onmessage=" + va.toString()], { type: "text/javascript" }))) : new Worker(x(c) ? c : "worker/worker.js", { type: "module" }) } catch (e) { } return d }; function Q(a) {
if (!(this instanceof Q)) return new Q(a); var b = a.document || a.doc || a, c; this.K = []; this.h = []; this.A = []; this.register = v(); this.key = (c = b.key || b.id) && S(c, this.A) || "id"; this.m = u(a.fastupdate); this.C = (c = b.store) && !0 !== c && []; this.store = c && v(); this.I = (c = b.tag) && S(c, this.A); this.l = c && v(); this.cache = (c = a.cache) && new J(c); a.cache = !1; this.o = a.worker; this.async = !1; c = v(); let d = b.index || b.field || b; x(d) && (d = [d]); for (let e = 0, f, g; e < d.length; e++)f = d[e], x(f) || (g = f, f = f.field), g = C(g) ? Object.assign({}, a, g) : a,
this.o && (c[f] = new O(g), c[f].o || (this.o = !1)), this.o || (c[f] = new K(g, this.register)), this.K[e] = S(f, this.A), this.h[e] = f; if (this.C) for (a = b.store, x(a) && (a = [a]), b = 0; b < a.length; b++)this.C[b] = S(a[b], this.A); this.index = c
} function S(a, b) { const c = a.split(":"); let d = 0; for (let e = 0; e < c.length; e++)a = c[e], 0 <= a.indexOf("[]") && (a = a.substring(0, a.length - 2)) && (b[d] = !0), a && (c[d++] = a); d < c.length && (c.length = d); return 1 < d ? c : c[0] } function T(a, b) { if (x(b)) a = a[b]; else for (let c = 0; a && c < b.length; c++)a = a[b[c]]; return a }
function U(a, b, c, d, e) { a = a[e]; if (d === c.length - 1) b[e] = a; else if (a) if (a.constructor === Array) for (b = b[e] = Array(a.length), e = 0; e < a.length; e++)U(a, b, c, d, e); else b = b[e] || (b[e] = v()), e = c[++d], U(a, b, c, d, e) } function V(a, b, c, d, e, f, g, h) { if (a = a[g]) if (d === b.length - 1) { if (a.constructor === Array) { if (c[d]) { for (b = 0; b < a.length; b++)e.add(f, a[b], !0, !0); return } a = a.join(" ") } e.add(f, a, h, !0) } else if (a.constructor === Array) for (g = 0; g < a.length; g++)V(a, b, c, d, e, f, g, h); else g = b[++d], V(a, b, c, d, e, f, g, h) } t = Q.prototype;
t.add = function (a, b, c) {
C(a) && (b = a, a = T(b, this.key)); if (b && (a || 0 === a)) {
if (!c && this.register[a]) return this.update(a, b); for (let d = 0, e, f; d < this.h.length; d++)f = this.h[d], e = this.K[d], x(e) && (e = [e]), V(b, e, this.A, 0, this.index[f], a, e[0], c); if (this.I) { let d = T(b, this.I), e = v(); x(d) && (d = [d]); for (let f = 0, g, h; f < d.length; f++)if (g = d[f], !e[g] && (e[g] = 1, h = this.l[g] || (this.l[g] = []), !c || !h.includes(a))) if (h[h.length] = a, this.m) { const k = this.register[a] || (this.register[a] = []); k[k.length] = h } } if (this.store && (!c || !this.store[a])) {
let d;
if (this.C) { d = v(); for (let e = 0, f; e < this.C.length; e++)f = this.C[e], x(f) ? d[f] = b[f] : U(b, d, f, 0, f[0]) } this.store[a] = d || b
}
} return this
}; t.append = function (a, b) { return this.add(a, b, !0) }; t.update = function (a, b) { return this.remove(a).add(a, b) };
t.remove = function (a) { C(a) && (a = T(a, this.key)); if (this.register[a]) { for (var b = 0; b < this.h.length && (this.index[this.h[b]].remove(a, !this.o), !this.m); b++); if (this.I && !this.m) for (let c in this.l) { b = this.l[c]; const d = b.indexOf(a); -1 !== d && (1 < b.length ? b.splice(d, 1) : delete this.l[c]) } this.store && delete this.store[a]; delete this.register[a] } return this };
t.search = function (a, b, c, d) {
c || (!b && C(a) ? (c = a, a = "") : C(b) && (c = b, b = 0)); let e = [], f = [], g, h, k, m, n, w, q = 0; if (c) if (c.constructor === Array) k = c, c = null; else { a = c.query || a; k = (g = c.pluck) || c.index || c.field; m = c.tag; h = this.store && c.enrich; n = "and" === c.bool; b = c.limit || b || 100; w = c.offset || 0; if (m && (x(m) && (m = [m]), !a)) { for (let l = 0, p; l < m.length; l++)if (p = ya.call(this, m[l], b, w, h)) e[e.length] = p, q++; return q ? e : [] } x(k) && (k = [k]) } k || (k = this.h); n = n && (1 < k.length || m && 1 < m.length); const r = !d && (this.o || this.async) && []; for (let l = 0, p, z, B; l <
k.length; l++) { let A; z = k[l]; x(z) || (A = z, z = A.field, a = A.query || a, b = A.limit || b); if (r) r[l] = this.index[z].searchAsync(a, b, A || c); else { d ? p = d[l] : p = this.index[z].search(a, b, A || c); B = p && p.length; if (m && B) { const y = []; let H = 0; n && (y[0] = [p]); for (let X = 0, pa, R; X < m.length; X++)if (pa = m[X], B = (R = this.l[pa]) && R.length) H++, y[y.length] = n ? [R] : R; H && (p = n ? ma(y, b || 100, w || 0) : na(p, y), B = p.length) } if (B) f[q] = z, e[q++] = p; else if (n) return [] } } if (r) {
const l = this; return new Promise(function (p) {
Promise.all(r).then(function (z) {
p(l.search(a, b,
c, z))
})
})
} if (!q) return []; if (g && (!h || !this.store)) return e[0]; for (let l = 0, p; l < f.length; l++) { p = e[l]; p.length && h && (p = za.call(this, p)); if (g) return p; e[l] = { field: f[l], result: p } } return e
}; function ya(a, b, c, d) { let e = this.l[a], f = e && e.length - c; if (f && 0 < f) { if (f > b || c) e = e.slice(c, c + b); d && (e = za.call(this, e)); return { tag: a, result: e } } } function za(a) { const b = Array(a.length); for (let c = 0, d; c < a.length; c++)d = a[c], b[c] = { id: d, doc: this.store[d] }; return b } t.contain = function (a) { return !!this.register[a] }; t.get = function (a) { return this.store[a] };
t.set = function (a, b) { this.store[a] = b; return this }; t.searchCache = oa; t.export = function (a, b, c, d, e) { e || (e = 0); d || (d = 0); if (d < this.h.length) { const f = this.h[d], g = this.index[f]; b = this; setTimeout(function () { g.export(a, b, e ? f : "", d, e++) || (d++, e = 1, b.export(a, b, f, d, e)) }) } else { let f, g; switch (e) { case 1: f = "tag"; g = this.l; break; case 2: f = "store"; g = this.store; break; default: return }ra(a, this, c, f, d, e, g) } };
t.import = function (a, b) { if (b) switch (x(b) && (b = JSON.parse(b)), a) { case "tag": this.l = b; break; case "reg": this.m = !1; this.register = b; for (let d = 0, e; d < this.h.length; d++)e = this.index[this.h[d]], e.register = b, e.m = !1; break; case "store": this.store = b; break; default: a = a.split("."); const c = a[0]; a = a[1]; c && a && this.index[c].import(a, b) } }; la(Q.prototype); var Ba = { encode: Aa, F: !1, G: "" }; const Ca = [F("[\u00e0\u00e1\u00e2\u00e3\u00e4\u00e5]"), "a", F("[\u00e8\u00e9\u00ea\u00eb]"), "e", F("[\u00ec\u00ed\u00ee\u00ef]"), "i", F("[\u00f2\u00f3\u00f4\u00f5\u00f6\u0151]"), "o", F("[\u00f9\u00fa\u00fb\u00fc\u0171]"), "u", F("[\u00fd\u0177\u00ff]"), "y", F("\u00f1"), "n", F("[\u00e7c]"), "k", F("\u00df"), "s", F(" & "), " and "]; function Aa(a) { var b = a = "" + a; b.normalize && (b = b.normalize("NFD").replace(ea, "")); return ca.call(this, b.toLowerCase(), !a.normalize && Ca) }; var Ea = { encode: Da, F: !1, G: "strict" }; const Fa = /[^a-z0-9]+/, Ga = { b: "p", v: "f", w: "f", z: "s", x: "s", "\u00df": "s", d: "t", n: "m", c: "k", g: "k", j: "k", q: "k", i: "e", y: "e", u: "o" }; function Da(a) { a = Aa.call(this, a).join(" "); const b = []; if (a) { const c = a.split(Fa), d = c.length; for (let e = 0, f, g = 0; e < d; e++)if ((a = c[e]) && (!this.filter || !this.filter[a])) { f = a[0]; let h = Ga[f] || f, k = h; for (let m = 1; m < a.length; m++) { f = a[m]; const n = Ga[f] || f; n && n !== k && (h += n, k = n) } b[g++] = h } } return b }; var Ia = { encode: Ha, F: !1, G: "" }; const Ja = [F("ae"), "a", F("oe"), "o", F("sh"), "s", F("th"), "t", F("ph"), "f", F("pf"), "f", F("(?![aeo])h(?![aeo])"), "", F("(?!^[aeo])h(?!^[aeo])"), ""]; function Ha(a, b) { a && (a = Da.call(this, a).join(" "), 2 < a.length && (a = E(a, Ja)), b || (1 < a.length && (a = ha(a)), a && (a = a.split(" ")))); return a || [] }; var La = { encode: Ka, F: !1, G: "" }; const Ma = F("(?!\\b)[aeo]"); function Ka(a) { a && (a = Ha.call(this, a, !0), 1 < a.length && (a = a.replace(Ma, "")), 1 < a.length && (a = ha(a)), a && (a = a.split(" "))); return a || [] }; G["latin:default"] = ja; G["latin:simple"] = Ba; G["latin:balance"] = Ea; G["latin:advanced"] = Ia; G["latin:extra"] = La; const W = self; let Y; const Z = { Index: K, Document: Q, Worker: O, registerCharset: function (a, b) { G[a] = b }, registerLanguage: function (a, b) { ka[a] = b } }; (Y = W.define) && Y.amd ? Y([], function () { return Z }) : W.exports ? W.exports = Z : W.FlexSearch = Z;
}(this));
</script>
<script src="https://cdn.jsdelivr.net/npm/mark.js"></script>
<style>
/* CSS for highlighted text */
mark {
background-color: yellow;
color: black;
font-weight: bold;
}
</style>
<link href="https://cdn.jsdelivr.net/npm/prismjs@v1.x/themes/prism.css" rel="stylesheet" />
<style>
#progress-overlay {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.1);
display: flex;
justify-content: center;
align-items: center;
z-index: 9999;
}
#progress-bar {
height: 20px;
background-color: #f2f2f2;
border: 1px solid #ccc;
overflow: hidden;
}
#progress-bar .progress {
width: 0;
height: 100%;
background-color: #4caf50 !important;
}
</style>
<style>
/* html, body{
background-color: transparent;
}
input {
background-color: transparent;
} */
.search-container {
margin-bottom: 10px;
}
p {
line-height: 1.5;
}
h2 {
overflow-x: auto;
}
/* Styles for mobile devices */
@media (max-width: 767px) {
#progress-bar {
width: 60%;
}
/*(h2{
font-size: 27px;
}*/
html,
body {
margin-left: 3%;
margin-right: 3%;
}
.repository-url {
display: none;
}
.search-container {
border: 1px solid #ccc;
}
.right-half {
/*padding-left: 10px;*/
padding-right: 10px;
}
.left-half {
padding-left: 15px;
padding-right: 10px;
padding-top: 5px;
padding-bottom: 5px;
}
#searchResults,
#searchInput {
font-size: 14px;
}
#searchInput {
text-indent: 15px;
}
.searchItemInfo {
padding-left: 15px;
}
}
/* Styles for desktop devices */
@media (min-width: 768px) {
#progress-bar {
width: 50%;
}
#searchInput {
text-indent: 20px;
}
html,
body {
margin-left: 5%;
margin-right: 5%;
}
#searchResults,
#searchInput {
font-size: 17px;
}
.searchItemInfo {
padding-left: 20px;
}
.search-container {
display: flex;
border: 1px solid #ccc;
}
.left-half,
.right-half {
flex: 1;
overflow-x: auto;
}
.right-half {
padding-right: 20px;
}
.left-half {
padding-top: 5px;
padding-left: 20px;
padding-right: 20px;
}
}
.left-half {
background-color: #ccc;
}
.right-half {
background-color: #f1f1f1;
}
.highlight {
background-color: yellow !important;
color: black !important;
}
.monospace-text {
font-family: "Courier New", monospace;
color: #333;
}
/* useless now */
pre {
overflow-x: auto;
white-space: pre-wrap;
white-space: -moz-pre-wrap;
white-space: -pre-wrap;
white-space: -o-pre-wrap;
word-wrap: break-word;
}
.codelink:hover {
cursor: pointer;
}
.codelink {
word-wrap: break-word;
/* Allow long words to be broken and wrap onto the next line */
overflow-wrap: break-word
}
/*
.searchItemInfo:hover {
cursor: pointer;
}
.search-container:hover {
cursor: pointer;
}
*/
.searchItem {
padding: 10px;
padding-bottom: 0;
padding-top: 0;
border: 1px solid #ccc;
}
/*
.searchItemInfo {
padding: 10px;
}
*/
</style>
<style type="text/css">
html,
body {
/*max-width: 900px;*/
font-family: 'Roboto', sans-serif;
height: 100%;
/* margin: 0; */
padding: 0;
display: flex;
flex-direction: column;
}
.container {
display: flex;
flex-direction: column;
align-items: left;
justify-content: center;
text-align: left;
/* margin: 5%; */
margin-bottom: 10px;
margin-top: 1.5%;
}
input[type="text"] {
padding: 10px;
/*border-radius: 10px;*/
border: 1px solid #ccc;
}
ul {
flex: 1;
/* Fill the remaining space */
overflow-y: auto;
/* Enable vertical scrolling */
list-style: none;
padding: 0;
/* margin: 5%; */
margin-top: 0;
}
ul li {
background-color: #f2f2f2;
margin-bottom: 10px;
/*border-radius: 10px;*/
box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.1);
}
</style>
</head>
<body>
<div id="progress-overlay">
<div id="progress-bar">
<div class="progress"></div>
</div>
</div>
<script src="https://cdn.jsdelivr.net/npm/prismjs@v1.x/components/prism-core.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/prismjs@v1.x/plugins/autoloader/prism-autoloader.min.js"></script>
<header class="container">
<h2><a id="github-link" class="github-icon"><i class="fab fa-github"></i></a> Document Index<span
class="repository-url"> of:
<span id="partial-repository-url"></span></span></h2>
<!-- <div class="search-input-container"> -->
<input type="text" id="searchInput" placeholder="Search...">
<!-- deprecate button. use enter instead. -->
<!-- <button id="searchButton" type="button"> -->
<!-- <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-search"
viewBox="0 0 16 16">
<path
d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0" />
</svg>
</button>
</div> -->
</header>
<ul id="searchResults"></ul>
<script type="text/javascript" defer>
// TODO: paging
const RESULT_LIMIT = 15;
// const RESULT_LIMIT = 50;
const progressOverlay = document.getElementById('progress-overlay');
const progressBar = document.querySelector('.progress');
function navigateToPage(base_filepath, language_id, project_id, detail_filepath = "") {
// Use a relative path to navigate to a specific page
let page_param = "codeview.html";
let file_param = 'src' + base_filepath;
file_param = encodeURIComponent(file_param);
let language_param = language_id
let jump_link = `${page_param}?file=${file_param}&language=${language_id}&project=${project_id}`;
if (detail_filepath !== "") {
let location_range = detail_filepath.slice(base_filepath.length + 1);
let location_param = `mycode.${location_range}`;
jump_link = `${jump_link}#${location_param}`;
}
window.location.href = jump_link;
}
/*async function waitForDOMContentLoaded() {
return new Promise(resolve => {
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', resolve);
} else {
resolve();
}
});
}
*/
// Sample data for demonstration
// async function async_main() {
function async_main() {
//const metadata_req = await fetch("metadata.json")// load from server
//const metadata = JSON.parse(await metadata_req.text())
var xhr = new XMLHttpRequest();
xhr.open('GET', "metadata.json", false); // The third parameter is set to false for synchronous request
xhr.send(null);
const metadata = JSON.parse(xhr.responseText);
const github_url = metadata.url.full;
const project_id = metadata.project_name;
const github_partial_url = metadata.url.partial;
const file_mapping = metadata.file_mapping
const split_count = metadata.split_count
var data_var = {}
for (let i = 0; i < split_count; i++) {
//let data_req = await fetch(`data/${i}.json`)// load from server
//let data_part = JSON.parse(await data_req.text())
var xhr = new XMLHttpRequest();
xhr.open('GET', `data/${i}.json`, false); // The third parameter is set to false for synchronous request
xhr.send(null);
let data_part = JSON.parse(xhr.responseText);
//data_var = { ...data_part, ...data_var }
data_var = Object.assign(data_var, data_part)
const progressPercentage = ((i + 1) / split_count) * 100;
progressBar.style.width = `${progressPercentage}%`;
}
progressOverlay.style.display = 'none';
const data = data_var // obviously not constant.
// debugger
// Create a new FlexSearch instance with the required configuration
const doc = new FlexSearch.Document({
tokenize: "full",
document: {
id: "id",
index: ["content"]
}
});
// Add the data to the search index
Object.keys(data).forEach(id => {
doc.add(id, data[id]);
});
const englishSymbols = ["!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", "}", "~"];
function replaceAll(inputString, searchValue, replaceValue) {
return inputString.split(searchValue).join(replaceValue);
}
function getSubTerms(it) {
var m_str = it
for (const sym of englishSymbols) {
m_str = replaceAll(m_str, sym, " ");
}
return m_str.split(" ");
}
const searchInputElem = document.getElementById('searchInput');
var isSearchRunning = false;
var hasInputEvent = false;
var lastInputEventTime = new Date().getTime();
function searchInputEventListener() {
function capitalizeFirstLetter(text) {
return text.charAt(0).toUpperCase() + text.slice(1);
}
function mutateText(text) {
var lowerText = text.toLowerCase()
var upperText = text.toUpperCase()
var lowerCapitalized = capitalizeFirstLetter(lowerText)
return [text, lowerText, upperText, lowerCapitalized]
}
const searchTerm = searchInputElem.value;
const searchSubTerms = getSubTerms(searchTerm);
function getHighlightTerms(msubterms) {
var ret = []
for (var it of msubterms) {
if (it.length > 0) {
for (var mut of mutateText(it)) {
if (ret.indexOf(mut) == -1) {
ret.push(mut);
}
}
}
}
return ret;
}
const searchHighlightTerms = getHighlightTerms(searchSubTerms);
/*function highlightTerm(innerContent){
var newContent = innerContent;
for (var it of searchHighlightTerms){
newContent = newContent.replaceAll(it, `<span class="highlight">${it}</span>`);
}
return newContent;
}*/
const results = doc.search(searchTerm, RESULT_LIMIT); // Limiting to 5 results for demonstration
const detail_types = ["code", "comment"];
// Display the search results
const searchResults = document.getElementById('searchResults');
searchResults.innerHTML = '';
var searchResultItems = {};
var searchResultItemIds = [];
results.forEach(result => {
const field = result.field; // "content"
const ids = result.result;
ids.forEach(id => {
const data_type = data[id].type;
const file_id = data[id].file_id;
if (searchResultItems[file_id] === undefined) {
searchResultItemIds.push(file_id);
const file_metadata = file_mapping[file_id];
const entry_id = file_metadata.entry_id;
const summary = data[entry_id + 1].content;
const filepath = file_metadata.filepath;
const language_id = file_metadata.language_id;
searchResultItems[file_id] = { "header": { "summary": summary, "filepath": filepath }, "pairs": {}, "language_id": language_id };
}
if (detail_types.indexOf(data_type) != -1) {
const pair_id = ((data_type === "code") ? id : (id - 1));
if (searchResultItems[file_id].pairs[pair_id] === undefined) {
searchResultItems[file_id].pairs[pair_id] = { "left": data[pair_id].content, "right": data[pair_id - 1].content, "location": data[pair_id].location };
}
}
// listItem.textContent = `[${field}][${data_type}] ${data[id][field]}`;
// searchResults.appendChild(listItem);
});
});
for (var file_id of searchResultItemIds) {
const it = searchResultItems[file_id];
const item = document.createElement('li');
//const item = document.createElement('div');
item.className = "searchItem";
const search_header = document.createElement('div');
search_header.className = "searchItemInfo";
const location_p = document.createElement('p');
location_p.className = "monospace-text codelink";
location_p.innerHTML = (it.header.filepath.slice(1));
location_p.setAttribute('onclick', `navigateToPage(${JSON.stringify(it.header.filepath)}, ${JSON.stringify(it.language_id)}, ${JSON.stringify(project_id)})`)
search_header.appendChild(location_p);
var skip_summary = false;
if (Object.keys(it.pairs).length == 1) {
if (it.pairs[Object.keys(it.pairs)[0]].right == it.header.summary) {
skip_summary = true;
}
}
if (!skip_summary) {
const summary_p = document.createElement('p');
summary_p.innerHTML = (it.header.summary);
search_header.appendChild(summary_p);
}
//search_header.setAttribute('onclick', `navigateToPage(${JSON.stringify(it.header.filepath)}, ${JSON.stringify(it.language_id)})`)
item.appendChild(search_header);
for (var pair_id in it.pairs) {
const pair_container = document.createElement('div');
pair_container.className = "search-container";
const pair = it.pairs[pair_id];
const pair_item = document.createElement('div');
const pair_left = document.createElement('div');
pair_left.className = "left-half";
const code_location_p = document.createElement('p');
code_location_p.className = "monospace-text codelink";
code_location_p.innerHTML = (pair.location.slice(1));
code_location_p.setAttribute('onclick', `navigateToPage(${JSON.stringify(it.header.filepath)}, ${JSON.stringify(it.language_id)}, ${JSON.stringify(project_id)}, ${JSON.stringify(pair.location)})`)
pair_left.appendChild(code_location_p);
const comment_p = document.createElement('p');
comment_p.innerHTML = (pair.right);
pair_left.appendChild(comment_p);
const pair_right = document.createElement('div');
pair_right.className = "right-half";
const code_pre = document.createElement('pre');
const code_code = document.createElement('code');
code_code.className = `language-${it.language_id}`
// Set the "data-dependencies" attribute
// code_code.setAttribute('data-dependencies', it.language_id+"!");
code_code.textContent = pair.left;
code_pre.appendChild(code_code);
//code_pre.className = "monospace-text";
//code_pre.innerHTML = highlightTerm(pair.left);
pair_right.appendChild(code_pre);
pair_container.appendChild(pair_right);
pair_container.appendChild(pair_left);
//pair_container.setAttribute('onclick', `navigateToPage(${JSON.stringify(it.header.filepath)}, ${JSON.stringify(it.language_id)}, ${JSON.stringify(pair.location)})`)
item.appendChild(pair_container);
}
//li_elem = document.createElement('li');
//li_elem.appendChild(item);
//searchResults.appendChild(li_elem);
searchResults.appendChild(item);
}
Prism.highlightAllUnder(searchResults);
// let's try understand that.
const markInstance = new Mark(document.getElementById('searchResults'));
markInstance.unmark(); // Clear previous marks
markInstance.mark(searchHighlightTerms);
}
function getQueryParams() {
var search = window.location.search.substring(1); // Remove leading '?'
var queryParams = {};
search.split('&').forEach(function (pair) {
var parts = pair.split('=');
var key = decodeURIComponent(parts[0]);
var value = decodeURIComponent(parts[1]);
queryParams[key] = value;
});
return queryParams;
}
// Event listener for the search input
function registerSearchEventListener() {
// this is never called
document.getElementById("partial-repository-url").innerText = github_partial_url;
document.getElementById("github-link").setAttribute("href", github_url);
// get query parameters.
// function keyPressListener(event) {
// if (event.key === 'Enter') {
// searchInputEventListener();
// }
// }
// searchInputElem.addEventListener('keypress',
// // searchInputElem.addEventListener('input',
// // searchInputEventListener
// keyPressListener
// );
setInterval(() => {
if (hasInputEvent) {
let currentTime = new Date().getTime();
if ((currentTime - lastInputEventTime) > 500) // .5 sec
{
hasInputEvent = false;
if (!isSearchRunning) {
isSearchRunning = true;
searchInputEventListener();
isSearchRunning = false;
}
}
}
}, 100)
searchInputElem.addEventListener('input', () => {
hasInputEvent = true;
lastInputEventTime = new Date().getTime();
})
function setTextAndTriggerInputEvent(queryString) {
searchInputElem.value = queryString; // Set text into the input box
// searchInputElem.dispatchEvent(new Event('keypress')); // Fire an input event
// var enterKeyEvent = new KeyboardEvent('keypress', {
// key: 'Enter'
// });
// searchInputElem.dispatchEvent(enterKeyEvent);
hasInputEvent = true;
}
function displayFile(file_path) {
// TODO: handle file query string
}
//const queryParams = new URLSearchParams(window.location.search);
const queryParams = getQueryParams(window.location.search);
const query_from_url = queryParams.q;
const file_path_from_url = queryParams.file;
if (query_from_url != null || query_from_url != undefined) {
setTextAndTriggerInputEvent(query_from_url)
} else if (file_path_from_url != null || file_path_from_url != undefined) {
displayFile(file_path_from_url)
}
}
//document.addEventListener('DOMContentLoaded', registerSearchEventListener);
// waitForDOMContentLoaded();
registerSearchEventListener()
// console.log("event listener registered")
}
async_main()
</script>
</body>
</html>

85
docs/metadata.json Normal file
View File

@ -0,0 +1,85 @@
{
"url": {
"full": "https://github.com/james4ever0/CLIP",
"partial": "james4ever0/CLIP"
},
"file_mapping": {
"0": {
"filepath": "/README.md",
"entry_id": 0,
"language_id": "plain-text"
},
"1": {
"filepath": "/hubconf.py",
"entry_id": 16,
"language_id": "python"
},
"2": {
"filepath": "/model-card.md",
"entry_id": 22,
"language_id": "markdown"
},
"3": {
"filepath": "/requirements.txt",
"entry_id": 40,
"language_id": "plain-text"
},
"4": {
"filepath": "/setup.py",
"entry_id": 44,
"language_id": "python"
},
"5": {
"filepath": "/clip/__init__.py",
"entry_id": 48,
"language_id": "python"
},
"6": {
"filepath": "/clip/clip.py",
"entry_id": 52,
"language_id": "python"
},
"7": {
"filepath": "/clip/model.py",
"entry_id": 74,
"language_id": "python"
},
"8": {
"filepath": "/clip/simple_tokenizer.py",
"entry_id": 110,
"language_id": "python"
},
"9": {
"filepath": "/data/country211.md",
"entry_id": 122,
"language_id": "markdown"
},
"10": {
"filepath": "/data/rendered-sst2.md",
"entry_id": 126,
"language_id": "markdown"
},
"11": {
"filepath": "/data/yfcc100m.md",
"entry_id": 130,
"language_id": "markdown"
},
"12": {
"filepath": "/notebooks/Interacting_with_CLIP.py",
"entry_id": 134,
"language_id": "python"
},
"13": {
"filepath": "/notebooks/Prompt_Engineering_for_ImageNet.py",
"entry_id": 144,
"language_id": "python"
},
"14": {
"filepath": "/tests/test_consistency.py",
"entry_id": 152,
"language_id": "python"
}
},
"project_name": "CLIP",
"split_count": 2
}

199
docs/src/README.md Normal file
View File

@ -0,0 +1,199 @@
# CLIP
[[Blog]](https://openai.com/blog/clip/) [[Paper]](https://arxiv.org/abs/2103.00020) [[Model Card]](model-card.md) [[Colab]](https://colab.research.google.com/github/openai/clip/blob/master/notebooks/Interacting_with_CLIP.ipynb)
CLIP (Contrastive Language-Image Pre-Training) is a neural network trained on a variety of (image, text) pairs. It can be instructed in natural language to predict the most relevant text snippet, given an image, without directly optimizing for the task, similarly to the zero-shot capabilities of GPT-2 and 3. We found CLIP matches the performance of the original ResNet50 on ImageNet “zero-shot” without using any of the original 1.28M labeled examples, overcoming several major challenges in computer vision.
## Approach
![CLIP](CLIP.png)
## Usage
First, [install PyTorch 1.7.1](https://pytorch.org/get-started/locally/) (or later) and torchvision, as well as small additional dependencies, and then install this repo as a Python package. On a CUDA GPU machine, the following will do the trick:
```bash
$ conda install --yes -c pytorch pytorch=1.7.1 torchvision cudatoolkit=11.0
$ pip install ftfy regex tqdm
$ pip install git+https://github.com/openai/CLIP.git
```
Replace `cudatoolkit=11.0` above with the appropriate CUDA version on your machine or `cpuonly` when installing on a machine without a GPU.
```python
import torch
import clip
from PIL import Image
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)
image = preprocess(Image.open("CLIP.png")).unsqueeze(0).to(device)
text = clip.tokenize(["a diagram", "a dog", "a cat"]).to(device)
with torch.no_grad():
image_features = model.encode_image(image)
text_features = model.encode_text(text)
logits_per_image, logits_per_text = model(image, text)
probs = logits_per_image.softmax(dim=-1).cpu().numpy()
print("Label probs:", probs) # prints: [[0.9927937 0.00421068 0.00299572]]
```
## API
The CLIP module `clip` provides the following methods:
#### `clip.available_models()`
Returns the names of the available CLIP models.
#### `clip.load(name, device=..., jit=False)`
Returns the model and the TorchVision transform needed by the model, specified by the model name returned by `clip.available_models()`. It will download the model as necessary. The `name` argument can also be a path to a local checkpoint.
The device to run the model can be optionally specified, and the default is to use the first CUDA device if there is any, otherwise the CPU. When `jit` is `False`, a non-JIT version of the model will be loaded.
#### `clip.tokenize(text: Union[str, List[str]], context_length=77)`
Returns a LongTensor containing tokenized sequences of given text input(s). This can be used as the input to the model
---
The model returned by `clip.load()` supports the following methods:
#### `model.encode_image(image: Tensor)`
Given a batch of images, returns the image features encoded by the vision portion of the CLIP model.
#### `model.encode_text(text: Tensor)`
Given a batch of text tokens, returns the text features encoded by the language portion of the CLIP model.
#### `model(image: Tensor, text: Tensor)`
Given a batch of images and a batch of text tokens, returns two Tensors, containing the logit scores corresponding to each image and text input. The values are cosine similarities between the corresponding image and text features, times 100.
## More Examples
### Zero-Shot Prediction
The code below performs zero-shot prediction using CLIP, as shown in Appendix B in the paper. This example takes an image from the [CIFAR-100 dataset](https://www.cs.toronto.edu/~kriz/cifar.html), and predicts the most likely labels among the 100 textual labels from the dataset.
```python
import os
import clip
import torch
from torchvision.datasets import CIFAR100
# Load the model
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load('ViT-B/32', device)
# Download the dataset
cifar100 = CIFAR100(root=os.path.expanduser("~/.cache"), download=True, train=False)
# Prepare the inputs
image, class_id = cifar100[3637]
image_input = preprocess(image).unsqueeze(0).to(device)
text_inputs = torch.cat([clip.tokenize(f"a photo of a {c}") for c in cifar100.classes]).to(device)
# Calculate features
with torch.no_grad():
image_features = model.encode_image(image_input)
text_features = model.encode_text(text_inputs)
# Pick the top 5 most similar labels for the image
image_features /= image_features.norm(dim=-1, keepdim=True)
text_features /= text_features.norm(dim=-1, keepdim=True)
similarity = (100.0 * image_features @ text_features.T).softmax(dim=-1)
values, indices = similarity[0].topk(5)
# Print the result
print("\nTop predictions:\n")
for value, index in zip(values, indices):
print(f"{cifar100.classes[index]:>16s}: {100 * value.item():.2f}%")
```
The output will look like the following (the exact numbers may be slightly different depending on the compute device):
```
Top predictions:
snake: 65.31%
turtle: 12.29%
sweet_pepper: 3.83%
lizard: 1.88%
crocodile: 1.75%
```
Note that this example uses the `encode_image()` and `encode_text()` methods that return the encoded features of given inputs.
### Linear-probe evaluation
The example below uses [scikit-learn](https://scikit-learn.org/) to perform logistic regression on image features.
```python
import os
import clip
import torch
import numpy as np
from sklearn.linear_model import LogisticRegression
from torch.utils.data import DataLoader
from torchvision.datasets import CIFAR100
from tqdm import tqdm
# Load the model
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load('ViT-B/32', device)
# Load the dataset
root = os.path.expanduser("~/.cache")
train = CIFAR100(root, download=True, train=True, transform=preprocess)
test = CIFAR100(root, download=True, train=False, transform=preprocess)
def get_features(dataset):
all_features = []
all_labels = []
with torch.no_grad():
for images, labels in tqdm(DataLoader(dataset, batch_size=100)):
features = model.encode_image(images.to(device))
all_features.append(features)
all_labels.append(labels)
return torch.cat(all_features).cpu().numpy(), torch.cat(all_labels).cpu().numpy()
# Calculate the image features
train_features, train_labels = get_features(train)
test_features, test_labels = get_features(test)
# Perform logistic regression
classifier = LogisticRegression(random_state=0, C=0.316, max_iter=1000, verbose=1)
classifier.fit(train_features, train_labels)
# Evaluate using the logistic regression classifier
predictions = classifier.predict(test_features)
accuracy = np.mean((test_labels == predictions).astype(float)) * 100.
print(f"Accuracy = {accuracy:.3f}")
```
Note that the `C` value should be determined via a hyperparameter sweep using a validation split.
## See Also
* [OpenCLIP](https://github.com/mlfoundations/open_clip): includes larger and independently trained CLIP models up to ViT-G/14
* [Hugging Face implementation of CLIP](https://huggingface.co/docs/transformers/model_doc/clip): for easier integration with the HF ecosystem

1
docs/src/clip/__init__.py Normal file
View File

@ -0,0 +1 @@
from .clip import *

245
docs/src/clip/clip.py Normal file
View File

@ -0,0 +1,245 @@
import hashlib
import os
import urllib
import warnings
from typing import Any, Union, List
from pkg_resources import packaging
import torch
from PIL import Image
from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
from tqdm import tqdm
from .model import build_model
from .simple_tokenizer import SimpleTokenizer as _Tokenizer
try:
from torchvision.transforms import InterpolationMode
BICUBIC = InterpolationMode.BICUBIC
except ImportError:
BICUBIC = Image.BICUBIC
if packaging.version.parse(torch.__version__) < packaging.version.parse("1.7.1"):
warnings.warn("PyTorch version 1.7.1 or higher is recommended")
__all__ = ["available_models", "load", "tokenize"]
_tokenizer = _Tokenizer()
_MODELS = {
"RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt",
"RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt",
"RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt",
"RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt",
"RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt",
"ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
"ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt",
"ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt",
"ViT-L/14@336px": "https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt",
}
def _download(url: str, root: str):
os.makedirs(root, exist_ok=True)
filename = os.path.basename(url)
expected_sha256 = url.split("/")[-2]
download_target = os.path.join(root, filename)
if os.path.exists(download_target) and not os.path.isfile(download_target):
raise RuntimeError(f"{download_target} exists and is not a regular file")
if os.path.isfile(download_target):
if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
return download_target
else:
warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")
with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop:
while True:
buffer = source.read(8192)
if not buffer:
break
output.write(buffer)
loop.update(len(buffer))
if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match")
return download_target
def _convert_image_to_rgb(image):
return image.convert("RGB")
def _transform(n_px):
return Compose([
Resize(n_px, interpolation=BICUBIC),
CenterCrop(n_px),
_convert_image_to_rgb,
ToTensor(),
Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
])
def available_models() -> List[str]:
"""Returns the names of available CLIP models"""
return list(_MODELS.keys())
def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None):
"""Load a CLIP model
Parameters
----------
name : str
A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
device : Union[str, torch.device]
The device to put the loaded model
jit : bool
Whether to load the optimized JIT model or more hackable non-JIT model (default).
download_root: str
path to download the model files; by default, it uses "~/.cache/clip"
Returns
-------
model : torch.nn.Module
The CLIP model
preprocess : Callable[[PIL.Image], torch.Tensor]
A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
"""
if name in _MODELS:
model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip"))
elif os.path.isfile(name):
model_path = name
else:
raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
with open(model_path, 'rb') as opened_file:
try:
# loading JIT archive
model = torch.jit.load(opened_file, map_location=device if jit else "cpu").eval()
state_dict = None
except RuntimeError:
# loading saved state dict
if jit:
warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
jit = False
state_dict = torch.load(opened_file, map_location="cpu")
if not jit:
model = build_model(state_dict or model.state_dict()).to(device)
if str(device) == "cpu":
model.float()
return model, _transform(model.visual.input_resolution)
# patch the device names
device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
def _node_get(node: torch._C.Node, key: str):
"""Gets attributes of a node which is polymorphic over return type.
From https://github.com/pytorch/pytorch/pull/82628
"""
sel = node.kindOf(key)
return getattr(node, sel)(key)
def patch_device(module):
try:
graphs = [module.graph] if hasattr(module, "graph") else []
except RuntimeError:
graphs = []
if hasattr(module, "forward1"):
graphs.append(module.forward1.graph)
for graph in graphs:
for node in graph.findAllNodes("prim::Constant"):
if "value" in node.attributeNames() and str(_node_get(node, "value")).startswith("cuda"):
node.copyAttributes(device_node)
model.apply(patch_device)
patch_device(model.encode_image)
patch_device(model.encode_text)
# patch dtype to float32 on CPU
if str(device) == "cpu":
float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
float_node = float_input.node()
def patch_float(module):
try:
graphs = [module.graph] if hasattr(module, "graph") else []
except RuntimeError:
graphs = []
if hasattr(module, "forward1"):
graphs.append(module.forward1.graph)
for graph in graphs:
for node in graph.findAllNodes("aten::to"):
inputs = list(node.inputs())
for i in [1, 2]: # dtype can be the second or third argument to aten::to()
if _node_get(inputs[i].node(), "value") == 5:
inputs[i].node().copyAttributes(float_node)
model.apply(patch_float)
patch_float(model.encode_image)
patch_float(model.encode_text)
model.float()
return model, _transform(model.input_resolution.item())
def tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]:
"""
Returns the tokenized representation of given input string(s)
Parameters
----------
texts : Union[str, List[str]]
An input string or a list of input strings to tokenize
context_length : int
The context length to use; all CLIP models use 77 as the context length
truncate: bool
Whether to truncate the text in case its encoding is longer than the context length
Returns
-------
A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length].
We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long.
"""
if isinstance(texts, str):
texts = [texts]
sot_token = _tokenizer.encoder["<|startoftext|>"]
eot_token = _tokenizer.encoder["<|endoftext|>"]
all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]
if packaging.version.parse(torch.__version__) < packaging.version.parse("1.8.0"):
result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
else:
result = torch.zeros(len(all_tokens), context_length, dtype=torch.int)
for i, tokens in enumerate(all_tokens):
if len(tokens) > context_length:
if truncate:
tokens = tokens[:context_length]
tokens[-1] = eot_token
else:
raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}")
result[i, :len(tokens)] = torch.tensor(tokens)
return result

436
docs/src/clip/model.py Normal file
View File

@ -0,0 +1,436 @@
from collections import OrderedDict
from typing import Tuple, Union
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1):
super().__init__()
# all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
self.bn1 = nn.BatchNorm2d(planes)
self.relu1 = nn.ReLU(inplace=True)
self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
self.bn2 = nn.BatchNorm2d(planes)
self.relu2 = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
self.bn3 = nn.BatchNorm2d(planes * self.expansion)
self.relu3 = nn.ReLU(inplace=True)
self.downsample = None
self.stride = stride
if stride > 1 or inplanes != planes * Bottleneck.expansion:
# downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
self.downsample = nn.Sequential(OrderedDict([
("-1", nn.AvgPool2d(stride)),
("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),
("1", nn.BatchNorm2d(planes * self.expansion))
]))
def forward(self, x: torch.Tensor):
identity = x
out = self.relu1(self.bn1(self.conv1(x)))
out = self.relu2(self.bn2(self.conv2(out)))
out = self.avgpool(out)
out = self.bn3(self.conv3(out))
if self.downsample is not None:
identity = self.downsample(x)
out += identity
out = self.relu3(out)
return out
class AttentionPool2d(nn.Module):
def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
super().__init__()
self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
self.k_proj = nn.Linear(embed_dim, embed_dim)
self.q_proj = nn.Linear(embed_dim, embed_dim)
self.v_proj = nn.Linear(embed_dim, embed_dim)
self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
self.num_heads = num_heads
def forward(self, x):
x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC
x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC
x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC
x, _ = F.multi_head_attention_forward(
query=x[:1], key=x, value=x,
embed_dim_to_check=x.shape[-1],
num_heads=self.num_heads,
q_proj_weight=self.q_proj.weight,
k_proj_weight=self.k_proj.weight,
v_proj_weight=self.v_proj.weight,
in_proj_weight=None,
in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
bias_k=None,
bias_v=None,
add_zero_attn=False,
dropout_p=0,
out_proj_weight=self.c_proj.weight,
out_proj_bias=self.c_proj.bias,
use_separate_proj_weight=True,
training=self.training,
need_weights=False
)
return x.squeeze(0)
class ModifiedResNet(nn.Module):
"""
A ResNet class that is similar to torchvision's but contains the following changes:
- There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
- Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
- The final pooling layer is a QKV attention instead of an average pool
"""
def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):
super().__init__()
self.output_dim = output_dim
self.input_resolution = input_resolution
# the 3-layer stem
self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
self.bn1 = nn.BatchNorm2d(width // 2)
self.relu1 = nn.ReLU(inplace=True)
self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)
self.bn2 = nn.BatchNorm2d(width // 2)
self.relu2 = nn.ReLU(inplace=True)
self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)
self.bn3 = nn.BatchNorm2d(width)
self.relu3 = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(2)
# residual layers
self._inplanes = width # this is a *mutable* variable used during construction
self.layer1 = self._make_layer(width, layers[0])
self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
embed_dim = width * 32 # the ResNet feature dimension
self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)
def _make_layer(self, planes, blocks, stride=1):
layers = [Bottleneck(self._inplanes, planes, stride)]
self._inplanes = planes * Bottleneck.expansion
for _ in range(1, blocks):
layers.append(Bottleneck(self._inplanes, planes))
return nn.Sequential(*layers)
def forward(self, x):
def stem(x):
x = self.relu1(self.bn1(self.conv1(x)))
x = self.relu2(self.bn2(self.conv2(x)))
x = self.relu3(self.bn3(self.conv3(x)))
x = self.avgpool(x)
return x
x = x.type(self.conv1.weight.dtype)
x = stem(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.attnpool(x)
return x
class LayerNorm(nn.LayerNorm):
"""Subclass torch's LayerNorm to handle fp16."""
def forward(self, x: torch.Tensor):
orig_type = x.dtype
ret = super().forward(x.type(torch.float32))
return ret.type(orig_type)
class QuickGELU(nn.Module):
def forward(self, x: torch.Tensor):
return x * torch.sigmoid(1.702 * x)
class ResidualAttentionBlock(nn.Module):
def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None):
super().__init__()
self.attn = nn.MultiheadAttention(d_model, n_head)
self.ln_1 = LayerNorm(d_model)
self.mlp = nn.Sequential(OrderedDict([
("c_fc", nn.Linear(d_model, d_model * 4)),
("gelu", QuickGELU()),
("c_proj", nn.Linear(d_model * 4, d_model))
]))
self.ln_2 = LayerNorm(d_model)
self.attn_mask = attn_mask
def attention(self, x: torch.Tensor):
self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None
return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
def forward(self, x: torch.Tensor):
x = x + self.attention(self.ln_1(x))
x = x + self.mlp(self.ln_2(x))
return x
class Transformer(nn.Module):
def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None):
super().__init__()
self.width = width
self.layers = layers
self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)])
def forward(self, x: torch.Tensor):
return self.resblocks(x)
class VisionTransformer(nn.Module):
def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int):
super().__init__()
self.input_resolution = input_resolution
self.output_dim = output_dim
self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
scale = width ** -0.5
self.class_embedding = nn.Parameter(scale * torch.randn(width))
self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
self.ln_pre = LayerNorm(width)
self.transformer = Transformer(width, layers, heads)
self.ln_post = LayerNorm(width)
self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
def forward(self, x: torch.Tensor):
x = self.conv1(x) # shape = [*, width, grid, grid]
x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2]
x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width]
x = x + self.positional_embedding.to(x.dtype)
x = self.ln_pre(x)
x = x.permute(1, 0, 2) # NLD -> LND
x = self.transformer(x)
x = x.permute(1, 0, 2) # LND -> NLD
x = self.ln_post(x[:, 0, :])
if self.proj is not None:
x = x @ self.proj
return x
class CLIP(nn.Module):
def __init__(self,
embed_dim: int,
# vision
image_resolution: int,
vision_layers: Union[Tuple[int, int, int, int], int],
vision_width: int,
vision_patch_size: int,
# text
context_length: int,
vocab_size: int,
transformer_width: int,
transformer_heads: int,
transformer_layers: int
):
super().__init__()
self.context_length = context_length
if isinstance(vision_layers, (tuple, list)):
vision_heads = vision_width * 32 // 64
self.visual = ModifiedResNet(
layers=vision_layers,
output_dim=embed_dim,
heads=vision_heads,
input_resolution=image_resolution,
width=vision_width
)
else:
vision_heads = vision_width // 64
self.visual = VisionTransformer(
input_resolution=image_resolution,
patch_size=vision_patch_size,
width=vision_width,
layers=vision_layers,
heads=vision_heads,
output_dim=embed_dim
)
self.transformer = Transformer(
width=transformer_width,
layers=transformer_layers,
heads=transformer_heads,
attn_mask=self.build_attention_mask()
)
self.vocab_size = vocab_size
self.token_embedding = nn.Embedding(vocab_size, transformer_width)
self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))
self.ln_final = LayerNorm(transformer_width)
self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
self.initialize_parameters()
def initialize_parameters(self):
nn.init.normal_(self.token_embedding.weight, std=0.02)
nn.init.normal_(self.positional_embedding, std=0.01)
if isinstance(self.visual, ModifiedResNet):
if self.visual.attnpool is not None:
std = self.visual.attnpool.c_proj.in_features ** -0.5
nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:
for name, param in resnet_block.named_parameters():
if name.endswith("bn3.weight"):
nn.init.zeros_(param)
proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
attn_std = self.transformer.width ** -0.5
fc_std = (2 * self.transformer.width) ** -0.5
for block in self.transformer.resblocks:
nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
if self.text_projection is not None:
nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
def build_attention_mask(self):
# lazily create causal attention mask, with full attention between the vision tokens
# pytorch uses additive attention mask; fill with -inf
mask = torch.empty(self.context_length, self.context_length)
mask.fill_(float("-inf"))
mask.triu_(1) # zero out the lower diagonal
return mask
@property
def dtype(self):
return self.visual.conv1.weight.dtype
def encode_image(self, image):
return self.visual(image.type(self.dtype))
def encode_text(self, text):
x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model]
x = x + self.positional_embedding.type(self.dtype)
x = x.permute(1, 0, 2) # NLD -> LND
x = self.transformer(x)
x = x.permute(1, 0, 2) # LND -> NLD
x = self.ln_final(x).type(self.dtype)
# x.shape = [batch_size, n_ctx, transformer.width]
# take features from the eot embedding (eot_token is the highest number in each sequence)
x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
return x
def forward(self, image, text):
image_features = self.encode_image(image)
text_features = self.encode_text(text)
# normalized features
image_features = image_features / image_features.norm(dim=1, keepdim=True)
text_features = text_features / text_features.norm(dim=1, keepdim=True)
# cosine similarity as logits
logit_scale = self.logit_scale.exp()
logits_per_image = logit_scale * image_features @ text_features.t()
logits_per_text = logits_per_image.t()
# shape = [global_batch_size, global_batch_size]
return logits_per_image, logits_per_text
def convert_weights(model: nn.Module):
"""Convert applicable model parameters to fp16"""
def _convert_weights_to_fp16(l):
if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
l.weight.data = l.weight.data.half()
if l.bias is not None:
l.bias.data = l.bias.data.half()
if isinstance(l, nn.MultiheadAttention):
for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
tensor = getattr(l, attr)
if tensor is not None:
tensor.data = tensor.data.half()
for name in ["text_projection", "proj"]:
if hasattr(l, name):
attr = getattr(l, name)
if attr is not None:
attr.data = attr.data.half()
model.apply(_convert_weights_to_fp16)
def build_model(state_dict: dict):
vit = "visual.proj" in state_dict
if vit:
vision_width = state_dict["visual.conv1.weight"].shape[0]
vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
image_resolution = vision_patch_size * grid_size
else:
counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
vision_layers = tuple(counts)
vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
vision_patch_size = None
assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
image_resolution = output_width * 32
embed_dim = state_dict["text_projection"].shape[1]
context_length = state_dict["positional_embedding"].shape[0]
vocab_size = state_dict["token_embedding.weight"].shape[0]
transformer_width = state_dict["ln_final.weight"].shape[0]
transformer_heads = transformer_width // 64
transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith("transformer.resblocks")))
model = CLIP(
embed_dim,
image_resolution, vision_layers, vision_width, vision_patch_size,
context_length, vocab_size, transformer_width, transformer_heads, transformer_layers
)
for key in ["input_resolution", "context_length", "vocab_size"]:
if key in state_dict:
del state_dict[key]
convert_weights(model)
model.load_state_dict(state_dict)
return model.eval()

132
docs/src/clip/simple_tokenizer.py Normal file
View File

@ -0,0 +1,132 @@
import gzip
import html
import os
from functools import lru_cache
import ftfy
import regex as re
@lru_cache()
def default_bpe():
return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
@lru_cache()
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
def get_pairs(word):
"""Return set of symbol pairs in a word.
Word is represented as tuple of symbols (symbols being variable-length strings).
"""
pairs = set()
prev_char = word[0]
for char in word[1:]:
pairs.add((prev_char, char))
prev_char = char
return pairs
def basic_clean(text):
text = ftfy.fix_text(text)
text = html.unescape(html.unescape(text))
return text.strip()
def whitespace_clean(text):
text = re.sub(r'\s+', ' ', text)
text = text.strip()
return text
class SimpleTokenizer(object):
def __init__(self, bpe_path: str = default_bpe()):
self.byte_encoder = bytes_to_unicode()
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
merges = merges[1:49152-256-2+1]
merges = [tuple(merge.split()) for merge in merges]
vocab = list(bytes_to_unicode().values())
vocab = vocab + [v+'</w>' for v in vocab]
for merge in merges:
vocab.append(''.join(merge))
vocab.extend(['<|startoftext|>', '<|endoftext|>'])
self.encoder = dict(zip(vocab, range(len(vocab))))
self.decoder = {v: k for k, v in self.encoder.items()}
self.bpe_ranks = dict(zip(merges, range(len(merges))))
self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}
self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
def bpe(self, token):
if token in self.cache:
return self.cache[token]
word = tuple(token[:-1]) + ( token[-1] + '</w>',)
pairs = get_pairs(word)
if not pairs:
return token+'</w>'
while True:
bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
if bigram not in self.bpe_ranks:
break
first, second = bigram
new_word = []
i = 0
while i < len(word):
try:
j = word.index(first, i)
new_word.extend(word[i:j])
i = j
except:
new_word.extend(word[i:])
break
if word[i] == first and i < len(word)-1 and word[i+1] == second:
new_word.append(first+second)
i += 2
else:
new_word.append(word[i])
i += 1
new_word = tuple(new_word)
word = new_word
if len(word) == 1:
break
else:
pairs = get_pairs(word)
word = ' '.join(word)
self.cache[token] = word
return word
def encode(self, text):
bpe_tokens = []
text = whitespace_clean(basic_clean(text)).lower()
for token in re.findall(self.pat, text):
token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
return bpe_tokens
def decode(self, tokens):
text = ''.join([self.decoder[token] for token in tokens])
text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
return text

12
docs/src/data/country211.md Normal file
View File

@ -0,0 +1,12 @@
# The Country211 Dataset
In the paper, we used an image classification dataset called Country211, to evaluate the model's capability on geolocation. To do so, we filtered the YFCC100m dataset that have GPS coordinate corresponding to a [ISO-3166 country code](https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes) and created a balanced dataset by sampling 150 train images, 50 validation images, and 100 test images images for each country.
The following command will download an 11GB archive countaining the images and extract into a subdirectory `country211`:
```bash
wget https://openaipublic.azureedge.net/clip/data/country211.tgz
tar zxvf country211.tgz
```
These images are a subset of the YFCC100m dataset. Use of the underlying media files is subject to the Creative Commons licenses chosen by their creators/uploaders. For more information about the YFCC100M dataset, visit [the official website](https://multimediacommons.wordpress.com/yfcc100m-core-dataset/).

11
docs/src/data/rendered-sst2.md Normal file
View File

@ -0,0 +1,11 @@
# The Rendered SST2 Dataset
In the paper, we used an image classification dataset called Rendered SST2, to evaluate the model's capability on optical character recognition. To do so, we rendered the sentences in the [Standford Sentiment Treebank v2](https://nlp.stanford.edu/sentiment/treebank.html) dataset and used those as the input to the CLIP image encoder.
The following command will download a 131MB archive countaining the images and extract into a subdirectory `rendered-sst2`:
```bash
wget https://openaipublic.azureedge.net/clip/data/rendered-sst2.tgz
tar zxvf rendered-sst2.tgz
```

14
docs/src/data/yfcc100m.md Normal file
View File

@ -0,0 +1,14 @@
# The YFCC100M Subset
In the paper, we performed a dataset ablation using a subset of the YFCC100M dataset and showed that the performance remained largely similar.
The subset contains 14,829,396 images, about 15% of the full dataset, which have been filtered to only keep those with natural languag titles and/or descriptions in English.
We provide the list of (line number, photo identifier, photo hash) of each image contained in this subset. These correspond to the first three columns in the dataset's metadata TSV file.
```bash
wget https://openaipublic.azureedge.net/clip/data/yfcc100m_subset_data.tsv.bz2
bunzip2 yfcc100m_subset_data.tsv.bz2
```
Use of the underlying media files is subject to the Creative Commons licenses chosen by their creators/uploaders. For more information about the YFCC100M dataset, visit [the official website](https://multimediacommons.wordpress.com/yfcc100m-core-dataset/).

42
docs/src/hubconf.py Normal file
View File

@ -0,0 +1,42 @@
from clip.clip import tokenize as _tokenize, load as _load, available_models as _available_models
import re
import string
dependencies = ["torch", "torchvision", "ftfy", "regex", "tqdm"]
# For compatibility (cannot include special characters in function name)
model_functions = { model: re.sub(f'[{string.punctuation}]', '_', model) for model in _available_models()}
def _create_hub_entrypoint(model):
def entrypoint(**kwargs):
return _load(model, **kwargs)
entrypoint.__doc__ = f"""Loads the {model} CLIP model
Parameters
----------
device : Union[str, torch.device]
The device to put the loaded model
jit : bool
Whether to load the optimized JIT model or more hackable non-JIT model (default).
download_root: str
path to download the model files; by default, it uses "~/.cache/clip"
Returns
-------
model : torch.nn.Module
The {model} CLIP model
preprocess : Callable[[PIL.Image], torch.Tensor]
A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
"""
return entrypoint
def tokenize():
return _tokenize
_entrypoints = {model_functions[model]: _create_hub_entrypoint(model) for model in _available_models()}
globals().update(_entrypoints)

120
docs/src/model-card.md Normal file
View File

@ -0,0 +1,120 @@
# Model Card: CLIP
Inspired by [Model Cards for Model Reporting (Mitchell et al.)](https://arxiv.org/abs/1810.03993) and [Lessons from Archives (Jo & Gebru)](https://arxiv.org/pdf/1912.10389.pdf), we’re providing some accompanying information about the multimodal model.
## Model Details
The CLIP model was developed by researchers at OpenAI to learn about what contributes to robustness in computer vision tasks. The model was also developed to test the ability of models to generalize to arbitrary image classification tasks in a zero-shot manner. It was not developed for general model deployment - to deploy models like CLIP, researchers will first need to carefully study their capabilities in relation to the specific context they’re being deployed within.
### Model Date
January 2021
### Model Type
The base model uses a ResNet50 with several modifications as an image encoder and uses a masked self-attention Transformer as a text encoder. These encoders are trained to maximize the similarity of (image, text) pairs via a contrastive loss. There is also a variant of the model where the ResNet image encoder is replaced with a Vision Transformer.
### Model Versions
Initially, we’ve released one CLIP model based on the Vision Transformer architecture equivalent to ViT-B/32, along with the RN50 model, using the architecture equivalent to ResNet-50.
As part of the staged release process, we have also released the RN101 model, as well as RN50x4, a RN50 scaled up 4x according to the [EfficientNet](https://arxiv.org/abs/1905.11946) scaling rule. In July 2021, we additionally released the RN50x16 and ViT-B/16 models, and in January 2022, the RN50x64 and ViT-L/14 models were released. Lastly, the ViT-L/14@336px model was released in April 2022.
Please see the paper linked below for further details about their specification.
### Documents
- [Blog Post](https://openai.com/blog/clip/)
- [CLIP Paper](https://arxiv.org/abs/2103.00020)
## Model Use
### Intended Use
The model is intended as a research output for research communities. We hope that this model will enable researchers to better understand and explore zero-shot, arbitrary image classification. We also hope it can be used for interdisciplinary studies of the potential impact of such models - the CLIP paper includes a discussion of potential downstream impacts to provide an example for this sort of analysis.
#### Primary intended uses
The primary intended users of these models are AI researchers.
We primarily imagine the model will be used by researchers to better understand robustness, generalization, and other capabilities, biases, and constraints of computer vision models.
### Out-of-Scope Use Cases
**Any** deployed use case of the model - whether commercial or not - is currently out of scope. Non-deployed use cases such as image search in a constrained environment, are also not recommended unless there is thorough in-domain testing of the model with a specific, fixed class taxonomy. This is because our safety assessment demonstrated a high need for task specific testing especially given the variability of CLIP’s performance with different class taxonomies. This makes untested and unconstrained deployment of the model in any use case currently potentially harmful.
Certain use cases which would fall under the domain of surveillance and facial recognition are always out-of-scope regardless of performance of the model. This is because the use of artificial intelligence for tasks such as these can be premature currently given the lack of testing norms and checks to ensure its fair use.
Since the model has not been purposefully trained in or evaluated on any languages other than English, its use should be limited to English language use cases.
## Data
The model was trained on publicly available image-caption data. This was done through a combination of crawling a handful of websites and using commonly-used pre-existing image datasets such as [YFCC100M](http://projects.dfki.uni-kl.de/yfcc100m/). A large portion of the data comes from our crawling of the internet. This means that the data is more representative of people and societies most connected to the internet which tend to skew towards more developed nations, and younger, male users.
### Data Mission Statement
Our goal with building this dataset was to test out robustness and generalizability in computer vision tasks. As a result, the focus was on gathering large quantities of data from different publicly-available internet data sources. The data was gathered in a mostly non-interventionist manner. However, we only crawled websites that had policies against excessively violent and adult images and allowed us to filter out such content. We do not intend for this dataset to be used as the basis for any commercial or deployed model and will not be releasing the dataset.
## Performance and Limitations
### Performance
We have evaluated the performance of CLIP on a wide range of benchmarks across a variety of computer vision datasets such as OCR to texture recognition to fine-grained classification. The paper describes model performance on the following datasets:
- Food101
- CIFAR10
- CIFAR100
- Birdsnap
- SUN397
- Stanford Cars
- FGVC Aircraft
- VOC2007
- DTD
- Oxford-IIIT Pet dataset
- Caltech101
- Flowers102
- MNIST
- SVHN
- IIIT5K
- Hateful Memes
- SST-2
- UCF101
- Kinetics700
- Country211
- CLEVR Counting
- KITTI Distance
- STL-10
- RareAct
- Flickr30
- MSCOCO
- ImageNet
- ImageNet-A
- ImageNet-R
- ImageNet Sketch
- ObjectNet (ImageNet Overlap)
- Youtube-BB
- ImageNet-Vid
## Limitations
CLIP and our analysis of it have a number of limitations. CLIP currently struggles with respect to certain tasks such as fine grained classification and counting objects. CLIP also poses issues with regards to fairness and bias which we discuss in the paper and briefly in the next section. Additionally, our approach to testing CLIP also has an important limitation- in many cases we have used linear probes to evaluate the performance of CLIP and there is evidence suggesting that linear probes can underestimate model performance.
### Bias and Fairness
We find that the performance of CLIP - and the specific biases it exhibits - can depend significantly on class design and the choices one makes for categories to include and exclude. We tested the risk of certain kinds of denigration with CLIP by classifying images of people from [Fairface](https://arxiv.org/abs/1908.04913) into crime-related and non-human animal categories. We found significant disparities with respect to race and gender. Additionally, we found that these disparities could shift based on how the classes were constructed. (Details captured in the Broader Impacts Section in the paper).
We also tested the performance of CLIP on gender, race and age classification using the Fairface dataset (We default to using race categories as they are constructed in the Fairface dataset.) in order to assess quality of performance across different demographics. We found accuracy >96% across all races for gender classification with ‘Middle Eastern’ having the highest accuracy (98.4%) and ‘White’ having the lowest (96.5%). Additionally, CLIP averaged ~93% for racial classification and ~63% for age classification. Our use of evaluations to test for gender, race and age classification as well as denigration harms is simply to evaluate performance of the model across people and surface potential risks and not to demonstrate an endorsement/enthusiasm for such tasks.
## Feedback
### Where to send questions or comments about the model
Please use [this Google Form](https://forms.gle/Uv7afRH5dvY34ZEs9)

143
docs/src/notebooks/Interacting_with_CLIP.py Normal file
View File

@ -0,0 +1,143 @@
#! pip install ftfy regex tqdm
#! pip install git+https://github.com/openai/CLIP.git
import numpy as np
import torch
from pkg_resources import packaging
print("Torch version:", torch.__version__)
import clip
clip.available_models()
model, preprocess = clip.load("ViT-B/32")
model.cuda().eval()
input_resolution = model.visual.input_resolution
context_length = model.context_length
vocab_size = model.vocab_size
print("Model parameters:", f"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}")
print("Input resolution:", input_resolution)
print("Context length:", context_length)
print("Vocab size:", vocab_size)
preprocess
clip.tokenize("Hello World!")
import os
import skimage
import IPython.display
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
from collections import OrderedDict
import torch
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
# images in skimage to use and their textual descriptions
descriptions = {
"page": "a page of text about segmentation",
"chelsea": "a facial photo of a tabby cat",
"astronaut": "a portrait of an astronaut with the American flag",
"rocket": "a rocket standing on a launchpad",
"motorcycle_right": "a red motorcycle standing in a garage",
"camera": "a person looking at a camera on a tripod",
"horse": "a black-and-white silhouette of a horse",
"coffee": "a cup of coffee on a saucer"
}
original_images = []
images = []
texts = []
plt.figure(figsize=(16, 5))
for filename in [filename for filename in os.listdir(skimage.data_dir) if filename.endswith(".png") or filename.endswith(".jpg")]:
name = os.path.splitext(filename)[0]
if name not in descriptions:
continue
image = Image.open(os.path.join(skimage.data_dir, filename)).convert("RGB")
plt.subplot(2, 4, len(images) + 1)
plt.imshow(image)
plt.title(f"{filename}\n{descriptions[name]}")
plt.xticks([])
plt.yticks([])
original_images.append(image)
images.append(preprocess(image))
texts.append(descriptions[name])
plt.tight_layout()
image_input = torch.tensor(np.stack(images)).cuda()
text_tokens = clip.tokenize(["This is " + desc for desc in texts]).cuda()
with torch.no_grad():
image_features = model.encode_image(image_input).float()
text_features = model.encode_text(text_tokens).float()
image_features /= image_features.norm(dim=-1, keepdim=True)
text_features /= text_features.norm(dim=-1, keepdim=True)
similarity = text_features.cpu().numpy() @ image_features.cpu().numpy().T
count = len(descriptions)
plt.figure(figsize=(20, 14))
plt.imshow(similarity, vmin=0.1, vmax=0.3)
# plt.colorbar()
plt.yticks(range(count), texts, fontsize=18)
plt.xticks([])
for i, image in enumerate(original_images):
plt.imshow(image, extent=(i - 0.5, i + 0.5, -1.6, -0.6), origin="lower")
for x in range(similarity.shape[1]):
for y in range(similarity.shape[0]):
plt.text(x, y, f"{similarity[y, x]:.2f}", ha="center", va="center", size=12)
for side in ["left", "top", "right", "bottom"]:
plt.gca().spines[side].set_visible(False)
plt.xlim([-0.5, count - 0.5])
plt.ylim([count + 0.5, -2])
plt.title("Cosine similarity between text and image features", size=20)
from torchvision.datasets import CIFAR100
cifar100 = CIFAR100(os.path.expanduser("~/.cache"), transform=preprocess, download=True)
text_descriptions = [f"This is a photo of a {label}" for label in cifar100.classes]
text_tokens = clip.tokenize(text_descriptions).cuda()
with torch.no_grad():
text_features = model.encode_text(text_tokens).float()
text_features /= text_features.norm(dim=-1, keepdim=True)
text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
top_probs, top_labels = text_probs.cpu().topk(5, dim=-1)
plt.figure(figsize=(16, 16))
for i, image in enumerate(original_images):
plt.subplot(4, 4, 2 * i + 1)
plt.imshow(image)
plt.axis("off")
plt.subplot(4, 4, 2 * i + 2)
y = np.arange(top_probs.shape[-1])
plt.grid()
plt.barh(y, top_probs[i])
plt.gca().invert_yaxis()
plt.gca().set_axisbelow(True)
plt.yticks(y, [cifar100.classes[index] for index in top_labels[i].numpy()])
plt.xlabel("probability")
plt.subplots_adjust(wspace=0.5)
plt.show()

84
docs/src/notebooks/Prompt_Engineering_for_ImageNet.py Normal file
View File

@ -0,0 +1,84 @@
#! pip install ftfy regex tqdm
#! pip install git+https://github.com/openai/CLIP.git
import numpy as np
import torch
import clip
from tqdm.notebook import tqdm
from pkg_resources import packaging
print("Torch version:", torch.__version__)
clip.available_models()
model, preprocess = clip.load("ViT-B/32")
input_resolution = model.visual.input_resolution
context_length = model.context_length
vocab_size = model.vocab_size
print("Model parameters:", f"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}")
print("Input resolution:", input_resolution)
print("Context length:", context_length)
print("Vocab size:", vocab_size)
import json
imagenet_data = json.loads(open("imagenet_data.json","r").read())
imagenet_classes = imagenet_data['imagenet_classes']
imagenet_templates = imagenet_data['imagenet_templates']
print(f"{len(imagenet_classes)} classes, {len(imagenet_templates)} templates")
# execute:
# ! pip install git+https://github.com/modestyachts/ImageNetV2_pytorch
from imagenetv2_pytorch import ImageNetV2Dataset
images = ImageNetV2Dataset(transform=preprocess)
loader = torch.utils.data.DataLoader(images, batch_size=32, num_workers=2)
def zeroshot_classifier(classnames, templates):
with torch.no_grad():
zeroshot_weights = []
for classname in tqdm(classnames):
texts = [template.format(classname) for template in templates] #format with class
texts = clip.tokenize(texts).cuda() #tokenize
class_embeddings = model.encode_text(texts) #embed with text encoder
class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)
class_embedding = class_embeddings.mean(dim=0)
class_embedding /= class_embedding.norm()
zeroshot_weights.append(class_embedding)
zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()
return zeroshot_weights
zeroshot_weights = zeroshot_classifier(imagenet_classes, imagenet_templates)
def accuracy(output, target, topk=(1,)):
pred = output.topk(max(topk), 1, True, True)[1].t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]
with torch.no_grad():
top1, top5, n = 0., 0., 0.
for i, (images, target) in enumerate(tqdm(loader)):
images = images.cuda()
target = target.cuda()
# predict
image_features = model.encode_image(images)
image_features /= image_features.norm(dim=-1, keepdim=True)
logits = 100. * image_features @ zeroshot_weights
# measure accuracy
acc1, acc5 = accuracy(logits, target, topk=(1, 5))
top1 += acc1
top5 += acc5
n += images.size(0)
top1 = (top1 / n) * 100
top5 = (top5 / n) * 100
print(f"Top-1 accuracy: {top1:.2f}")
print(f"Top-5 accuracy: {top5:.2f}")

5
docs/src/requirements.txt Normal file
View File

@ -0,0 +1,5 @@
ftfy
regex
tqdm
torch
torchvision

21
docs/src/setup.py Normal file
View File

@ -0,0 +1,21 @@
import os
import pkg_resources
from setuptools import setup, find_packages
setup(
name="clip",
py_modules=["clip"],
version="1.0",
description="",
author="OpenAI",
packages=find_packages(exclude=["tests*"]),
install_requires=[
str(r)
for r in pkg_resources.parse_requirements(
open(os.path.join(os.path.dirname(__file__), "requirements.txt"))
)
],
include_package_data=True,
extras_require={'dev': ['pytest']},
)

25
docs/src/tests/test_consistency.py Normal file
View File

@ -0,0 +1,25 @@
import numpy as np
import pytest
import torch
from PIL import Image
import clip
@pytest.mark.parametrize('model_name', clip.available_models())
def test_consistency(model_name):
device = "cpu"
jit_model, transform = clip.load(model_name, device=device, jit=True)
py_model, _ = clip.load(model_name, device=device, jit=False)
image = transform(Image.open("CLIP.png")).unsqueeze(0).to(device)
text = clip.tokenize(["a diagram", "a dog", "a cat"]).to(device)
with torch.no_grad():
logits_per_image, _ = jit_model(image, text)
jit_probs = logits_per_image.softmax(dim=-1).cpu().numpy()
logits_per_image, _ = py_model(image, text)
py_probs = logits_per_image.softmax(dim=-1).cpu().numpy()
assert np.allclose(jit_probs, py_probs, atol=0.01, rtol=0.1)

143
notebooks/Interacting_with_CLIP.py Normal file
View File

@ -0,0 +1,143 @@
#! pip install ftfy regex tqdm
#! pip install git+https://github.com/openai/CLIP.git
import numpy as np
import torch
from pkg_resources import packaging
print("Torch version:", torch.__version__)
import clip
clip.available_models()
model, preprocess = clip.load("ViT-B/32")
model.cuda().eval()
input_resolution = model.visual.input_resolution
context_length = model.context_length
vocab_size = model.vocab_size
print("Model parameters:", f"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}")
print("Input resolution:", input_resolution)
print("Context length:", context_length)
print("Vocab size:", vocab_size)
preprocess
clip.tokenize("Hello World!")
import os
import skimage
import IPython.display
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
from collections import OrderedDict
import torch
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
# images in skimage to use and their textual descriptions
descriptions = {
"page": "a page of text about segmentation",
"chelsea": "a facial photo of a tabby cat",
"astronaut": "a portrait of an astronaut with the American flag",
"rocket": "a rocket standing on a launchpad",
"motorcycle_right": "a red motorcycle standing in a garage",
"camera": "a person looking at a camera on a tripod",
"horse": "a black-and-white silhouette of a horse",
"coffee": "a cup of coffee on a saucer"
}
original_images = []
images = []
texts = []
plt.figure(figsize=(16, 5))
for filename in [filename for filename in os.listdir(skimage.data_dir) if filename.endswith(".png") or filename.endswith(".jpg")]:
name = os.path.splitext(filename)[0]
if name not in descriptions:
continue
image = Image.open(os.path.join(skimage.data_dir, filename)).convert("RGB")
plt.subplot(2, 4, len(images) + 1)
plt.imshow(image)
plt.title(f"{filename}\n{descriptions[name]}")
plt.xticks([])
plt.yticks([])
original_images.append(image)
images.append(preprocess(image))
texts.append(descriptions[name])
plt.tight_layout()
image_input = torch.tensor(np.stack(images)).cuda()
text_tokens = clip.tokenize(["This is " + desc for desc in texts]).cuda()
with torch.no_grad():
image_features = model.encode_image(image_input).float()
text_features = model.encode_text(text_tokens).float()
image_features /= image_features.norm(dim=-1, keepdim=True)
text_features /= text_features.norm(dim=-1, keepdim=True)
similarity = text_features.cpu().numpy() @ image_features.cpu().numpy().T
count = len(descriptions)
plt.figure(figsize=(20, 14))
plt.imshow(similarity, vmin=0.1, vmax=0.3)
# plt.colorbar()
plt.yticks(range(count), texts, fontsize=18)
plt.xticks([])
for i, image in enumerate(original_images):
plt.imshow(image, extent=(i - 0.5, i + 0.5, -1.6, -0.6), origin="lower")
for x in range(similarity.shape[1]):
for y in range(similarity.shape[0]):
plt.text(x, y, f"{similarity[y, x]:.2f}", ha="center", va="center", size=12)
for side in ["left", "top", "right", "bottom"]:
plt.gca().spines[side].set_visible(False)
plt.xlim([-0.5, count - 0.5])
plt.ylim([count + 0.5, -2])
plt.title("Cosine similarity between text and image features", size=20)
from torchvision.datasets import CIFAR100
cifar100 = CIFAR100(os.path.expanduser("~/.cache"), transform=preprocess, download=True)
text_descriptions = [f"This is a photo of a {label}" for label in cifar100.classes]
text_tokens = clip.tokenize(text_descriptions).cuda()
with torch.no_grad():
text_features = model.encode_text(text_tokens).float()
text_features /= text_features.norm(dim=-1, keepdim=True)
text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
top_probs, top_labels = text_probs.cpu().topk(5, dim=-1)
plt.figure(figsize=(16, 16))
for i, image in enumerate(original_images):
plt.subplot(4, 4, 2 * i + 1)
plt.imshow(image)
plt.axis("off")
plt.subplot(4, 4, 2 * i + 2)
y = np.arange(top_probs.shape[-1])
plt.grid()
plt.barh(y, top_probs[i])
plt.gca().invert_yaxis()
plt.gca().set_axisbelow(True)
plt.yticks(y, [cifar100.classes[index] for index in top_labels[i].numpy()])
plt.xlabel("probability")
plt.subplots_adjust(wspace=0.5)
plt.show()

84
notebooks/Prompt_Engineering_for_ImageNet.py Normal file
View File

@ -0,0 +1,84 @@
#! pip install ftfy regex tqdm
#! pip install git+https://github.com/openai/CLIP.git
import numpy as np
import torch
import clip
from tqdm.notebook import tqdm
from pkg_resources import packaging
print("Torch version:", torch.__version__)
clip.available_models()
model, preprocess = clip.load("ViT-B/32")
input_resolution = model.visual.input_resolution
context_length = model.context_length
vocab_size = model.vocab_size
print("Model parameters:", f"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}")
print("Input resolution:", input_resolution)
print("Context length:", context_length)
print("Vocab size:", vocab_size)
import json
imagenet_data = json.loads(open("imagenet_data.json","r").read())
imagenet_classes = imagenet_data['imagenet_classes']
imagenet_templates = imagenet_data['imagenet_templates']
print(f"{len(imagenet_classes)} classes, {len(imagenet_templates)} templates")
# execute:
# ! pip install git+https://github.com/modestyachts/ImageNetV2_pytorch
from imagenetv2_pytorch import ImageNetV2Dataset
images = ImageNetV2Dataset(transform=preprocess)
loader = torch.utils.data.DataLoader(images, batch_size=32, num_workers=2)
def zeroshot_classifier(classnames, templates):
with torch.no_grad():
zeroshot_weights = []
for classname in tqdm(classnames):
texts = [template.format(classname) for template in templates] #format with class
texts = clip.tokenize(texts).cuda() #tokenize
class_embeddings = model.encode_text(texts) #embed with text encoder
class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)
class_embedding = class_embeddings.mean(dim=0)
class_embedding /= class_embedding.norm()
zeroshot_weights.append(class_embedding)
zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()
return zeroshot_weights
zeroshot_weights = zeroshot_classifier(imagenet_classes, imagenet_templates)
def accuracy(output, target, topk=(1,)):
pred = output.topk(max(topk), 1, True, True)[1].t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]
with torch.no_grad():
top1, top5, n = 0., 0., 0.
for i, (images, target) in enumerate(tqdm(loader)):
images = images.cuda()
target = target.cuda()
# predict
image_features = model.encode_image(images)
image_features /= image_features.norm(dim=-1, keepdim=True)
logits = 100. * image_features @ zeroshot_weights
# measure accuracy
acc1, acc5 = accuracy(logits, target, topk=(1, 5))
top1 += acc1
top5 += acc5
n += images.size(0)
top1 = (top1 / n) * 100
top5 = (top5 / n) * 100
print(f"Top-1 accuracy: {top1:.2f}")
print(f"Top-5 accuracy: {top5:.2f}")

1086
notebooks/imagenet_data.json Normal file
View File

File diff suppressed because it is too large Load Diff