← Back to all posts

Great. i was wondering if you could help me with a few things. i will make sure to read and try all of your responses(if there are any!).
1. can you make it so that you can create your own username
and it re does all of the previous code except the create
2. Add a piece of code that checks for the letter press and
if it is the same as the letter on screen, it goes on to
the next letter(a lot like nitrotype.com)
3. A POINT SYSTEM. i have tried that so many time that it is actually ridiculous.
thanks if you at least try and evon more thanks if you can do even one of the three things i asked.
👍 🎉 THANKS 🎉 👍
P.S.
go onto files an click nitrotype

Answered by AlexanderTarn (290) [earned 5 cycles]
hotnewtop
AlexanderTarn (290)

you can make a file with username list, you then check if the username is in there and use information from that and put it into your code.
you can use readchar() to get an instant input of 1 letter, which you can use for a code like in nitrotype. Point system is easy, just have a variable and when you get the right thing just += 1 the variable.

code5 (15)

or this:

``````import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from functools import partial
from itertools import chain
from revtorch import ReversibleBlock, ReversibleSequence

# helper fns

def sort_key_val(t1, t2, dim=-1):
values, indices = t1.sort(dim=dim)
t2 = t2.expand_as(t1)
return values, t2.gather(dim, indices)

def batched_index_select(values, indices):
last_dim = values.shape[-1]
return values.gather(1, indices[:, :, None].expand(-1, -1, last_dim))

def process_inputs_chunk(fn, *args, chunks=1):
chunked_inputs = list(map(lambda x: x.chunk(chunks, dim=0), args))
outputs = [fn(*input_pair) for input_pair in zip(*chunked_inputs)]
return outputs

def chunked_sum(tensor, chunks=1):
*orig_size, last_dim = tensor.shape
tensor = tensor.reshape(-1, last_dim)
summed_tensors = [c.sum(dim=-1) for c in tensor.chunk(chunks, dim=0)]

def cache_fn(f):
cache = None
def cached_fn(*args, **kwargs):
nonlocal cache
if cache is not None:
return cache
cache = f(*args, **kwargs)
return cache
return cached_fn

def default(val, default_val):
return default_val if val is None else val

# helper classes

class ScaleNorm(nn.Module):
def __init__(self, emb, eps=1e-5):
super().__init__()
self.eps = eps

def forward(self, x):
n = torch.norm(x, dim=-1, keepdim=True).clamp(min=self.eps)
return x / n * self.g

class WithNorm(nn.Module):
def __init__(self, norm_class, emb, fn):
super().__init__()
self.emb = emb
self.norm = norm_class(emb)
self.fn = fn
def forward(self, x):
x = self.norm(x)
return self.fn(x)

class Chunk(nn.Module):
def __init__(self, chunks, fn, along_dim = -1):
super().__init__()
self.dim = along_dim
self.chunks = chunks
self.fn = fn

def forward(self, x):
chunks = x.chunk(self.chunks, dim = self.dim)

class SettableArgs(nn.Module):
def __init__(self, fn, *args, **kwargs):
super().__init__()
self.args = args
self.kwargs = kwargs
self.fn = fn

def set_args(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs

def forward(self, x):
return self.fn(x, *self.args, **self.kwargs)

# LSH attention as described in https://openreview.net/pdf?id=rkgNKkHtvB
# adapted from trax, stripped to what paper said needed to work
# namely that buckets need to be at least 64 with 8 rounds of hashing

class LSHAttention(nn.Module):
def __init__( self,
dropout = 0.,
bucket_size = 64,
n_hashes = 8,
causal = False,
allow_duplicate_attention = True,
attend_across_buckets = True,
rehash_each_round = True,
drop_for_hash_rate = 0.0,
super().__init__()
if dropout >= 1.0:
raise ValueError('Dropout rates must be lower than 1.')

self.dropout = nn.Dropout(dropout)
self.dropout_for_hash = nn.Dropout(drop_for_hash_rate)

assert rehash_each_round or allow_duplicate_attention, (
'The setting {allow_duplicate_attention=False, rehash_each_round=False}'
' is not implemented.')

self.causal = causal
self.n_hashes = n_hashes
self.bucket_size = bucket_size

self._allow_duplicate_attention = allow_duplicate_attention
self._attend_across_buckets = attend_across_buckets
self._rehash_each_round = rehash_each_round

def hash_vectors(self, n_buckets, vecs):
batch_size = vecs.shape
device = vecs.device

# See https://arxiv.org/pdf/1509.02897.pdf
# We sample a different random rotation for each round of hashing to
# decrease the probability of hash misses.
assert n_buckets % 2 == 0

rot_size = n_buckets

rotations_shape = (
vecs.shape[-1],
self.n_hashes if self._rehash_each_round else 1,
rot_size // 2)

random_rotations = torch.randn(rotations_shape, device=device).expand(batch_size, -1, -1, -1)

dropped_vecs = self.dropout_for_hash(vecs)
rotated_vecs = torch.einsum('btf,bfhi->bhti', dropped_vecs, random_rotations)

if self._rehash_each_round:
rotated_vecs = torch.cat([rotated_vecs, -rotated_vecs], dim=-1)
buckets = torch.argmax(rotated_vecs, dim=-1)
# buckets is now (self.n_hashes, seqlen). Next we add offsets so that
# bucket numbers from different hashing rounds don't overlap.
offsets = torch.arange(self.n_hashes, device=device)
offsets = torch.reshape(offsets * n_buckets, (1, -1, 1))
buckets = torch.reshape(buckets + offsets, (batch_size, -1,))
else:
rotated_vecs = torch.cat([rotated_vecs, -rotated_vecs], dim=-1)
# In this configuration, we map each item to the top self.n_hashes buckets
rotated_vecs = torch.squeeze(rotated_vecs, 0)
bucket_range = torch.arange(rotated_vecs.shape[-1], device=device)
bucket_range = torch.reshape(bucket_range, (1, -1))
bucket_range = bucket_range.expand_as(rotated_vecs.shape)

_, buckets = sort_key_val(rotated_vecs, bucket_range, dim=-1)
buckets = buckets[:, -self.n_hashes:]

h, *_ = buckets.shape
buckets = torch.reshape(buckets.permute((*_, h)), (-1,))

return buckets

def forward(self, qk, v, query_len = None):
batch_size, seqlen, dim = qk.shape
query_len = default(query_len, seqlen)
device = qk.device

n_buckets = seqlen // self.bucket_size

buckets = self.hash_vectors(n_buckets, qk)
# We use the same vector as both a query and a key.
assert int(buckets.shape) == self.n_hashes * seqlen

ticker = torch.arange(self.n_hashes * seqlen, device=device).unsqueeze(0).expand_as(buckets)
buckets_and_t = seqlen * buckets + (ticker % seqlen)
buckets_and_t = buckets_and_t.detach()

# Hash-based sort ("s" at the start of variable names means "sorted")
sbuckets_and_t, sticker = sort_key_val(buckets_and_t, ticker, dim=-1)
_, undo_sort = sort_key_val(sticker, ticker, dim=-1)
del ticker

sbuckets_and_t = sbuckets_and_t.detach()
sticker = sticker.detach()
undo_sort = undo_sort.detach()

st = (sticker % seqlen)
sqk = batched_index_select(qk, st)
sv = batched_index_select(v, st)

# Split off a "bin" axis so that attention only occurs within chunks.
chunk_size = self.n_hashes * n_buckets
bq_t = bkv_t = torch.reshape(st, (batch_size, chunk_size, -1))
bqk = torch.reshape(sqk, (batch_size, chunk_size, -1, dim))
bv = torch.reshape(sv, (batch_size, chunk_size, -1, dim))
bq_buckets = bkv_buckets = torch.reshape(sbuckets_and_t // seqlen, (batch_size, chunk_size, -1))

# Hashing operates on unit-length vectors. Unnormalized query vectors are
# fine because they effectively provide a learnable temperature for the
# attention softmax, but normalizing keys is needed so that similarity for
# the purposes of attention correctly corresponds to hash locality.
bq = bqk
bk = F.normalize(bqk, p=2, dim=-1)

# Allow each chunk to attend within itself, and also one chunk back. Chunk
# boundaries might occur in the middle of a sequence of items from the
# same bucket, so this increases the chances of attending to relevant items.
def look_one_back(x):
x_extra = torch.cat([x[:, -1:, ...], x[:, :-1, ...]], dim=1)

bk = look_one_back(bk)
bv = look_one_back(bv)
bkv_t = look_one_back(bkv_t)
bkv_buckets = look_one_back(bkv_buckets)

# Dot-product attention.
dots = torch.einsum('bhie,bhje->bhij', bq, bk) * (dim ** -0.5)

if self.causal:
mask = bq_t[:, :, :, None] < bkv_t[:, :, None, :].clamp(max=query_len - 1)

# Mask out attention to self except when no other targets are available.
self_mask = bq_t[:, :, :, None] == bkv_t[:, :, None, :]

# Mask out attention to other hash buckets.
if not self._attend_across_buckets:
bucket_mask = bq_buckets[:, :, :, None] != bkv_buckets[:, :, None, :]

# Don't double-count query-key pairs across multiple rounds of hashing.
# There are two possible strategies here. (1) The default is to count how
# many times a query-key pair is repeated, and to lower its log-prob
# correspondingly at each repetition. (2) When hard_k is set, the code
# instead masks all but the first occurence of each query-key pair.
if not self._allow_duplicate_attention:
locs1 = undo_sort // bq_t.shape[-1]
locs2 = (locs1 + 1) % chunk_size
if not self._attend_across_buckets:
locs1 = buckets * chunk_size + locs1
locs2 = buckets * chunk_size + locs2
locs = torch.cat([
torch.reshape(locs1, (batch_size, self.n_hashes, seqlen)),
torch.reshape(locs2, (batch_size, self.n_hashes, seqlen)),
], 1).permute((0, 2, 1))

slocs = batched_index_select(locs, st)
b_locs = torch.reshape(slocs, (batch_size, chunk_size, -1, 2 * self.n_hashes))

b_locs1 = b_locs[:, :, :, None, :self.n_hashes]

bq_locs = b_locs1.expand(b_locs.shape[:3] + (2, self.n_hashes))
bq_locs = torch.reshape(bq_locs, b_locs.shape)
bkv_locs = look_one_back(b_locs)

dup_counts = (bq_locs[:, :, :, None, :] == bkv_locs[:, :, None, :, :])
# for memory considerations, chunk summation of last dimension for counting duplicates
dup_counts = chunked_sum(dup_counts, chunks=(self.n_hashes * batch_size))
dup_counts = dup_counts.detach()
assert dup_counts.shape == dots.shape
dots = dots - torch.log(dup_counts + 1e-9)
del dup_counts

# Softmax.
dots_logsumexp = torch.logsumexp(dots, dim=-1, keepdim=True)
dots = torch.exp(dots - dots_logsumexp)
dots = self.dropout(dots)

bo = torch.einsum('buij,buje->buie', dots, bv)
so = torch.reshape(bo, (batch_size, -1, dim))
slogits = torch.reshape(dots_logsumexp, (batch_size, -1,))

class UnsortLogits(Function):
@staticmethod
def forward(ctx, so, slogits):
so = so.detach()
slogits = slogits.detach()
o = batched_index_select(so, undo_sort)
_, logits = sort_key_val(sticker, slogits, dim=-1)
return o, logits

@staticmethod

o, logits = UnsortLogits.apply(so, slogits)
o = torch.reshape(o, (batch_size, self.n_hashes, seqlen, dim))
logits = torch.reshape(logits, (batch_size, self.n_hashes, seqlen, 1))

if query_len != seqlen:
query_slice = (slice(None), slice(None), slice(0, query_len))
o, logits = o[query_slice], logits[query_slice]

if self.n_hashes == 1:
out = o.squeeze(1)
else:
probs = torch.exp(logits - torch.logsumexp(logits, dim=1, keepdim=True))
out = torch.sum(o * probs, dim=1)

return out, buckets

class LSHSelfAttention(nn.Module):
def __init__(self, emb, heads = 8, bucket_size = 64, n_hashes = 8, causal = False, attn_chunks = None, random_rotations_per_head = False, attend_across_buckets = True, allow_duplicate_attention = True, num_mem_kv = 0, **kwargs):
super().__init__()
assert emb % heads == 0, 'dimensions must be divisible by number of heads'

self.emb = emb

self.toqk = nn.Linear(emb, emb, bias = False)
self.tov = nn.Linear(emb, emb, bias = False)
self.to_out = nn.Linear(emb, emb)

self.bucket_size = bucket_size
self.lsh_attn = LSHAttention(bucket_size=bucket_size, n_hashes=n_hashes, causal=causal, random_rotations_per_head=random_rotations_per_head, attend_across_buckets = attend_across_buckets,  allow_duplicate_attention = allow_duplicate_attention, **kwargs)

self.num_mem_kv = num_mem_kv
self.mem_kv = nn.Parameter(torch.randn(1, num_mem_kv, emb, requires_grad=True))

def forward(self, x, keys = None):
device = x.device
b, t, e, h, m = *x.shape, self.heads, self.num_mem_kv

mem = self.mem_kv.expand(b, m, e)
keys = default(keys, torch.empty(b, 0, e, device=device))

kv_len = t + m + keys.shape
assert kv_len % self.bucket_size == 0, f'Sequence length needs to be divisible by target bucket size - {self.bucket_size}'

x = torch.cat((x, mem, keys), dim=1)
qk = self.toqk(x)
v = self.tov(x)

return v.view(b, kv_len, h, -1).transpose(1, 2).reshape(b * h, kv_len, -1)

return v.view(b, h, t, -1).transpose(1, 2).contiguous()

lsh_attn_fn = partial(self.lsh_attn, query_len = t)
outputs = process_inputs_chunk(lsh_attn_fn, qk, v, chunks=self.attn_chunks)
attn_out = torch.cat([output for (output, _) in outputs], dim=0)

return self.to_out(out)

# simple full self attention for ablation

class SelfAttention(nn.Module):
def __init__(self, emb, heads = 8, causal = False, num_mem_kv = 0):
super().__init__()
assert emb % heads == 0, 'dimensions must be divisible by number of heads'
self.to_out = nn.Linear(emb, emb)
self.causal = causal

self.num_mem_kv = num_mem_kv
self.mem_kv = nn.Parameter(torch.randn(num_mem_kv, 1, emb, requires_grad=True))

def forward(self, x, keys = None):
device = x.device
b, t, e, m = *x.shape, self.num_mem_kv

mem = self.mem_kv.expand(m, b, e)
keys = default(keys, torch.empty(b, 0, e, device=device))

x, keys = x.transpose(0, 1), keys.transpose(0, 1)

kv = torch.cat((x, mem, keys))

attn_shape = (t, kv.shape)
if self.causal:
i, j = torch.triu_indices(t, t, 1)

return self.to_out(output.transpose(0, 1))

# feedforward

class GELU(nn.Module):
def forward(self, x):
return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))

class FeedForward(nn.Module):
def __init__(self, emb, mult = 4):
super().__init__()
self.emb = emb
self.net = nn.Sequential(
nn.Linear(emb, emb * mult),
GELU(),
nn.Linear(emb * mult, emb)
)

def forward(self, x):
return self.net(x)

# reformer lm

class Reformer(nn.Module):
def __init__(self, emb, depth, max_seq_len, heads = 8, bucket_size = 64, n_hashes = 8, ff_chunks = 100, attn_chunks = None, causal = False, weight_tie = False, lsh_dropout = 0., lsh_attend_across_buckets = True, lsh_allow_duplicate_attention = True, random_rotations_per_head = False, twin_attention = False, use_scale_norm = False, use_full_attn = False, num_mem_kv = 0):
super().__init__()
self.emb = emb
self.depth = depth

get_full_attn = lambda: SettableArgs(SelfAttention(emb, heads, causal = causal, num_mem_kv = num_mem_kv))
get_lsh_attn = lambda: SettableArgs(LSHSelfAttention(emb, heads, bucket_size, n_hashes, causal = causal, dropout = lsh_dropout, attn_chunks = attn_chunks, allow_duplicate_attention = lsh_allow_duplicate_attention, attend_across_buckets = lsh_attend_across_buckets, random_rotations_per_head = random_rotations_per_head, num_mem_kv = num_mem_kv))

get_attn = get_full_attn if use_full_attn else get_lsh_attn
get_ff = lambda: FeedForward(emb)

if weight_tie:
get_attn = cache_fn(get_attn)
get_ff = cache_fn(get_ff)

blocks = []
norm_type = ScaleNorm if use_scale_norm else nn.LayerNorm

for _ in range(depth):
attn = get_attn()
parallel_net = get_attn() if twin_attention else get_ff()

f = WithNorm(norm_type, emb, attn)
g = WithNorm(norm_type, emb, parallel_net)

if not twin_attention and ff_chunks > 1:
g = Chunk(ff_chunks, g, along_dim = -2)

blocks.append(ReversibleBlock(f, g, split_along_dim=-1, fix_random_seed=True))

self.layers = ReversibleSequence(nn.ModuleList(blocks))
self.modules = list(chain(*[[m.f_block.fn, m.g_block.fn] for m in blocks]))

def set_reversible_args(self, *args, **kwargs):
for module in self.modules:
if isinstance(module, SettableArgs):
module.set_args(*args, **kwargs)

def forward(self, x, keys = None):
x = torch.cat([x, x], dim = -1)
self.set_reversible_args(keys = keys)
x = self.layers(x)

class ReformerLM(nn.Module):
def __init__(self, num_tokens, emb, depth, max_seq_len, heads = 8, bucket_size = 64, n_hashes = 8, ff_chunks = 100, attn_chunks = None, causal = False, weight_tie = False, lsh_dropout = 0., random_rotations_per_head = False, twin_attention = False, use_scale_norm = False, use_full_attn = False, num_mem_kv = 0):
super().__init__()
self.token_emb = nn.Embedding(num_tokens, emb)
self.pos_emb = nn.Embedding(max_seq_len, emb)
self.reformer = Reformer(emb, depth, max_seq_len, heads = heads, bucket_size = bucket_size, n_hashes = n_hashes, ff_chunks = ff_chunks, attn_chunks = attn_chunks, causal = causal, weight_tie = weight_tie, lsh_dropout = lsh_dropout, random_rotations_per_head = random_rotations_per_head, twin_attention = twin_attention, use_scale_norm = use_scale_norm, use_full_attn = use_full_attn, num_mem_kv = num_mem_kv)
self.to_logits = nn.Linear(emb, num_tokens)

def forward(self, x, **kwargs):
x = self.token_emb(x) + self.pos_emb(torch.arange(x.shape, device=x.device))
x = self.reformer(x, **kwargs)
return self.to_logits(x)``````
Imthebestthe (108)

@code5 thanks so much for dropping in. i don't really understand how it works or if i could implement it but thanks for the effort
👍👍
what language is it?????

Imthebestthe (108)

@code5 Traceback (most recent call last):
File "main.py", line 2, in <module>
import torch
ModuleNotFoundError: No module named 'torch'
how do i stop that???

CalebCarlson (31)

Repl.it doesn't support the torch library, that's why its not [email protected]

Nettakrim (694)

i sure do hate it when my full self attention for ablation isnt simple!

code5 (15)

you should write

``print('hello world')``
itsevanb (4)

@Imthebestthe Copying and pasting is not code

CalebCarlson (31)

The only problem with that is that you haven't even tried to put an hour, much less even one second into it.

code5 (15)

@itsevanb indeed it is not, and indeed it was clearly a joke. Please go see my profile of projects I have made. I am not that good but I am indeed getting somewhere with the language C# and Python! Thank you for pointing out the fact I copied and pasted it. Copyright 2019, MIT Licence, all rights reserved.

`````` import os, json, subprocess

_user_ = {}

def _gather_user_info_(info):
global _user_
_user_ = info

_gather_user_info_(information)

def _complete_user_setup_():
with open('user_info.json','w') as file:
to_json = json.dumps(_user_,indent=2,sort_keys=False)
file.write(to_json)
file.close()

print('Done!')

with open('user_info.json','w') as file:
to_json = json.dumps(_user_,indent=2,sort_keys=False)
file.write(to_json)
file.close()
else:
subprocess.call('exit 1',shell=True)
else:
print('There is no ' + username + ' in the user database. Sorry')
subprocess.call('exit 1', shell=True)

_complete_user_setup_()

subprocess.call('clear',shell=True)

This is a simple application i've made :)

Imthebestthe (108)

@itsevanb i did not coppy and past

thank you for pointing out that I plagiarize code

itsevanb (4)

@code5 Was not talking to you, but thanks for pointing that out!

TST2024 (12)

Hi @Imthebestthe I could help you through discord so I can understand what you are asking for. I used to be in the same position as you lol. Send me your discord and I will add you.

Imthebestthe (108)

@TST2024 i dont have a discord. how do u make one and can i just type without calls????

CalebCarlson (31)

yes, you can type in discord without a call.

Foster_Bryant (89)

Ik how to do all those things but how would I add a point system for a game that doesn't even exist yet...